Merge pull request #209 from poli0981/feat/pipeline-headroom

v4.4.0 — The Headroom Edition: data jobs that scale with the catalogue
This commit is contained in:
poli0981 authored and GitHub committed 2026-10-23 22:28:58 +07:00
commit 5eadfbf34e
57 files changed
+2339 -324

No files matched your search

-19
View File
@@ -1,19 +0,0 @@
name : Check Dead Links
on :
schedule : [ { cron : '0 4 1,6,11,16,21,26 * *' } ]
workflow_dispatch :
permissions : { contents : write }
concurrency :
group : data-write
cancel-in-progress : false
jobs :
check :
runs-on : ubuntu-latest
steps :
-
uses : actions/checkout@v6
-
uses : actions/setup-python@v6
with : { python-version : '3.15' }
-
run : chmod +x ./bash/dead_links.sh && ./bash/dead_links.sh
+15 -10
View File
@@ -5,26 +5,31 @@ on :
# fires the ingest on a detached HEAD, and dies with exit 128 on push.
#
# data/blocklist.jsonl is here too: this job also applies removals
# (bash/ingest.sh runs scripts/remove_games.py first). A separate removal
# workflow would share `data-write` with this one, and each would evict the
# other's pending run.
# (bash/ingest.sh runs scripts/remove_games.py first), so one run does both.
push : { branches : [ main ], paths : [ 'scripts/temp_info.jsonl', 'data/blocklist.jsonl' ] }
# Orphan recovery, not a poll. `data-write` keeps at most ONE run pending and
# cancels the older one, so a push here that queues behind a ~115-minute data
# job is silently evicted the moment any OTHER data-write workflow queues --
# and nothing retries it, leaving approved games sitting in temp_info.jsonl
# until the next unrelated push. Re-running on a clear file is free:
# ingest_new.py returns immediately and `git diff --staged --quiet` skips the
# commit, so this is idempotent by construction.
# A safety net, not a poll. Before data-write queued its runs (queue: max),
# a push here that waited behind a long data job was evicted the moment any
# other data-write workflow queued, and approved games sat in
# temp_info.jsonl until the next unrelated push. Nothing is evicted now, but
# a failed run still leaves its links queued. Re-running on a clear file is
# free: ingest_new.py returns immediately and `git diff --staged --quiet`
# skips the commit, so this is idempotent by construction.
schedule : [ { cron : '0 */3 * * *' } ]
workflow_dispatch :
permissions : { contents : write }
# queue: max - a run that finds another data job running waits its turn
# (first in, first out) instead of evicting the run already waiting.
concurrency :
group : data-write
cancel-in-progress : false
queue : max
jobs :
ingest :
runs-on : ubuntu-latest
# About seven seconds a game, and an /admin batch holds at most 100. No
# time budget: ingest_new.py empties the whole queue file when it is done,
# so a game it had not reached would be lost.
timeout-minutes : 120
steps :
-
uses : actions/checkout@v6
+4
View File
@@ -19,13 +19,17 @@ permissions:
contents: write
actions: write
# queue: max - a run that finds another data job running waits its turn
# (first in, first out) instead of evicting the run already waiting.
concurrency:
group: data-write
cancel-in-progress: false
queue: max
jobs:
mark:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
# The tip of main when the job starts, not when it was queued: a run
-1
View File
@@ -13,7 +13,6 @@ on:
- "Auto Update JSON Data"
- "Top Online Leaderboard"
- "Update Reviews"
- "Check Dead Links"
- "Purge Unhealthy Games"
- "Ingest New Game Links"
- "Force Re-fetch All (v2.1 Schema)"
+27 -2
View File
@@ -1,20 +1,45 @@
name : Purge Unhealthy Games
# Takes out the games that left the store, stopped being free or are not out
# yet - only when two of Steam's answers agree: IStoreBrowseService/GetItems
# screens the whole catalogue in a hundred-odd requests, and appdetails
# confirms each game it flags (scripts/purge_unhealthy.py has the rules and
# the guards). A few minutes, so it runs daily. It replaces both the old
# purge (appdetails for every game, 5.3 hours) and Check Dead Links, a 5.5-hour
# HEAD sweep that never found anything: Steam sends a delisted app's page to
# the store front, which answers 200.
on :
schedule : [ { cron : '0 5 * * 1' } ]
schedule : [ { cron : '0 7 * * *' } ]
workflow_dispatch :
inputs :
dry_run :
description : 'Dry run: screen and confirm, remove nothing'
type : boolean
default : false
permissions : { contents : write }
# queue: max - a run that finds another data job running waits its turn
# (first in, first out) instead of evicting the run already waiting.
concurrency :
group : data-write
cancel-in-progress : false
queue : max
jobs :
purge :
runs-on : ubuntu-latest
timeout-minutes : 60
steps :
-
uses : actions/checkout@v6
# main as it is when the job STARTS: a run that waited its turn must
# work on what the job before it committed.
with : { ref : main }
-
uses : actions/setup-python@v6
with : { python-version : '3.15' }
-
run : chmod +x ./bash/purge.sh && ./bash/purge.sh
env : { STEAM_API_KEY : '${{ secrets.STEAM_API_KEY }}' }
env :
STEAM_API_KEY : '${{ secrets.STEAM_API_KEY }}'
JOB_BUDGET_MINUTES : '40'
PURGE_DRY_RUN : ${{ inputs.dry_run }}
+22 -6
View File
@@ -1,16 +1,25 @@
name : Force Re-fetch All (v2.1 Schema)
# ⚠ MANUAL ONLY – this re-fetches ALL games from Steam.
# Takes a long time for large datasets (300+ games ≈ 15-30 min).
# Creates a backup at data.jsonl.bak before starting.
# ⚠ MANUAL ONLY – this re-fetches games from Steam from scratch, about six
# seconds a game: far longer than one job may run once the catalogue is in
# the thousands. Run it in PARTS: part "1/8" takes an eighth of the games
# (crc32 slices, scripts/refetch_all.py), and the runs for 1/8 ... 8/8 can be
# started together - the data-write group queues them (queue: max). A part cut
# short by its budget keeps what it did; starting it again carries on with the
# games it did not reach. Creates a backup at data.bak before starting.
on :
workflow_dispatch :
inputs :
confirm :
description : 'Type "yes" to confirm force re-fetch of ALL games'
description : 'Type "yes" to confirm force re-fetch'
required : true
type : string
part :
description : 'Which part, K/N (1/8 ... 8/8 for the whole catalogue)'
required : true
default : '1/8'
type : string
permissions :
contents : write
@@ -18,16 +27,21 @@ permissions :
concurrency :
group : data-write
cancel-in-progress : false
queue : max
jobs :
refetch :
if : ${{ github.event.inputs.confirm == 'yes' }}
runs-on : ubuntu-latest
timeout-minutes : 120
timeout-minutes : 350
steps :
-
name : Checkout repo
uses : actions/checkout@v6
# main as it is when the job STARTS: a part that waited its turn must
# work on what the part before it committed.
with :
ref : main
-
name : Setup Python
@@ -36,9 +50,11 @@ jobs :
python-version : '3.15'
-
name : Force re-fetch all games
name : Force re-fetch one part
env :
STEAM_API_KEY : ${{ secrets.STEAM_API_KEY }}
REFETCH_PART : ${{ inputs.part }}
JOB_BUDGET_MINUTES : '320'
run : |
chmod +x ./bash/refetch_all.sh
./bash/refetch_all.sh
+17 -10
View File
@@ -1,16 +1,18 @@
name: Refresh All-Time Peaks
# all_time_peak - the most players a game has had at once - from SteamCharts'
# recorded peak, a thirtieth of the catalogue a day (~186 pages, 2-3 s apart:
# about ten minutes), so every game is asked once a month. Every run also
# raises the field from our own Steam API samples, which needs no requests.
# scripts/refresh_peaks.py has the rules: the field only ever goes up, and a
# refusal from SteamCharts ends the run early with a warning, not a failure.
# recorded peak, a thirtieth of the catalogue a day (~330 pages, 2-3 s apart:
# about a quarter of an hour), so every game is asked once a month. Past 400
# pages a day the cycle grows to 60 days and more instead: never more often
# than once a month a game. Every run also raises the field from our own
# Steam API samples, which needs no requests. scripts/refresh_peaks.py has
# the rules: the field only ever goes up, and a refusal from SteamCharts - or
# the time budget running out - ends the run early with a warning, not a
# failure.
#
# 19:30 UTC sits between Refresh Store Data (16:00, ~40 minutes) and the Daily
# Snapshot (23:00), where only the short Ingest runs are due. Scheduled runs
# here start up to a few hours late, and the data-write group keeps only ONE
# pending run, so this stays clear of the long morning jobs.
# 19:30 UTC sits between Refresh Store Data (16:00, about an hour) and the
# Daily Snapshot (23:00). A data job still running only delays this one: the
# data-write group queues runs (queue: max) instead of evicting them.
#
# The one-time backfill is a manual run with the box ticked: every game, about
# four hours. Start it in the evening UTC, after Refresh Store Data.
@@ -30,15 +32,20 @@ permissions: {contents: write}
concurrency:
group: data-write
cancel-in-progress: false
queue: max
jobs:
refresh:
runs-on: ubuntu-latest
timeout-minutes: ${{ inputs.backfill && 330 || 30 }}
timeout-minutes: ${{ inputs.backfill && 340 || 45 }}
steps:
# main as it is when the job STARTS: a run that waited its turn must
# work on what the job before it committed.
- uses: actions/checkout@v6
with: {ref: main}
- uses: actions/setup-python@v6
with: {python-version: '3.15'}
- run: chmod +x ./bash/refresh_peaks.sh && ./bash/refresh_peaks.sh
env:
PEAKS_ALL: ${{ inputs.backfill }}
JOB_BUDGET_MINUTES: ${{ inputs.backfill && '300' || '25' }}
+13 -8
View File
@@ -5,22 +5,23 @@ name: Refresh Store Data
# store page, every game once a week; name, header image, description,
# developer/publisher, release date, platforms, Metacritic and DRM notes from
# appdetails, every game once a month. Each daily run takes one slice of each
# rotation - roughly 630 store pages and 150 appdetails calls, ~40 minutes.
# rotation - about 1,400 store pages and 330 appdetails calls at 9,800 games,
# an hour. Past 2,000 pages or 500 appdetails a day a cycle doubles instead of
# the run growing, so it stays under an hour and a half.
# scripts/refresh_store_data.py explains the rotation and the rules (never
# blanks a value, never touches MANUAL_FIELDS).
#
# Then the anti-cheat sweep, committed separately: every game's anti-cheat as
# its developer declares it on Steam, about 160 GetItems requests plus a store
# its developer declares it on Steam, about 200 GetItems requests plus a store
# page for each disclosure Steam does not name there (~70), a few minutes.
# scripts/refresh_anticheat.py has its rules: Steam wins, an override wins
# over Steam, and no disclosure never clears a value. `only: anticheat` runs
# just the sweep.
#
# 16:00 UTC is the quiet part of this repository's day. Scheduled runs here
# start two to five hours late, the morning data jobs (reviews, dead links,
# purge) run for two to four hours, and the data-write group keeps only ONE
# pending run: a second one queueing behind a running job cancels the first.
# Nothing else long is due this late; the Daily Snapshot (23:00) comes after.
# 16:00 UTC is the quiet part of this repository's day: scheduled runs here
# start up to an hour late, and the morning jobs are done by then. A data job
# that is still running only delays this one - the data-write group queues
# runs (queue: max) instead of evicting the one waiting.
on:
schedule: [{cron: '0 16 * * *'}]
@@ -38,11 +39,12 @@ permissions: {contents: write}
concurrency:
group: data-write
cancel-in-progress: false
queue: max
jobs:
refresh:
runs-on: ubuntu-latest
timeout-minutes: 120
timeout-minutes: 150
steps:
# main, not the triggering commit: a run that waited its turn in
# data-write must work on the data as it is now, not as it was queued.
@@ -53,3 +55,6 @@ jobs:
- run: chmod +x ./bash/refresh_store.sh && ./bash/refresh_store.sh
env:
REFRESH_ONLY: ${{ inputs.only }}
# The store refresh's budget (core/budget.py); the anti-cheat sweep
# after it takes a few minutes and has none.
JOB_BUDGET_MINUTES: '100'
+8 -1
View File
@@ -4,15 +4,22 @@ on:
workflow_dispatch:
permissions: {contents: write}
# Shares the data-write group with every other workflow that commits to
# data/ — without it this can push mid-run against a 2-hour job.
# data/ — without it this can push mid-run against a long job. queue: max -
# a run that finds another data job running waits its turn (first in, first
# out) instead of evicting the run already waiting.
concurrency:
group: data-write
cancel-in-progress: false
queue: max
jobs:
snapshot:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
# main as it is when the job STARTS, so the snapshot holds what the
# jobs before it committed.
- uses: actions/checkout@v4
with: {ref: main}
- uses: actions/setup-python@v5
with: {python-version: '3.15'}
- run: chmod +x ./bash/snapshot.sh && ./bash/snapshot.sh
+13 -1
View File
@@ -5,15 +5,27 @@ on:
schedule: [{cron: '0 5 1,15 * *'}]
workflow_dispatch:
permissions: {contents: write}
# queue: max - a run that finds another data job running waits its turn
# (first in, first out) instead of evicting the run already waiting.
concurrency:
group: data-write
cancel-in-progress: false
queue: max
jobs:
leaderboard:
runs-on: ubuntu-latest
# About 0.55 s a game through the keyed Web API: ~65 minutes for 7,000
# offline games, ~2 hours at 14,000. The budget stops the sampling with
# time left to save and commit.
timeout-minutes: 240
steps:
# main as it is when the job STARTS: a run that waited its turn must
# work on what the job before it committed.
- uses: actions/checkout@v4
with: {ref: main}
- uses: actions/setup-python@v5
with: {python-version: '3.15'}
- run: chmod +x ./bash/offline.sh && ./bash/offline.sh
env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'}
env:
STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'
JOB_BUDGET_MINUTES: '200'
+13 -1
View File
@@ -3,15 +3,27 @@ on:
schedule: [{cron: '0 3 * * 0,3,5'}]
workflow_dispatch:
permissions: {contents: write}
# queue: max - a run that finds another data job running waits its turn
# (first in, first out) instead of evicting the run already waiting.
concurrency:
group: data-write
cancel-in-progress: false
queue: max
jobs:
leaderboard:
runs-on: ubuntu-latest
# The player counts come from the keyed Web API with no pause between
# batches, about 0.55 s a game: ~27 minutes for 2,900 online games. The
# budget stops the sampling with time left to save and commit.
timeout-minutes: 150
steps:
# main as it is when the job STARTS: a run that waited its turn must
# work on what the job before it committed.
- uses: actions/checkout@v4
with: {ref: main}
- uses: actions/setup-python@v5
with: {python-version: '3.15'}
- run: chmod +x ./bash/online.sh && ./bash/online.sh
env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'}
env:
STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'
JOB_BUDGET_MINUTES: '120'
+12 -1
View File
@@ -3,15 +3,26 @@ on:
schedule: [{cron: '0 0 * * *'}]
workflow_dispatch:
permissions: {contents: write}
# queue: max - a run that finds another data job running waits its turn
# (first in, first out) instead of evicting the run already waiting.
concurrency:
group: data-write
cancel-in-progress: false
queue: max
jobs:
update:
runs-on: ubuntu-latest
timeout-minutes: 90
steps:
# main as it is when the job STARTS. The commit the scheduled run was
# created on is older whenever another data job ran first, and this one
# then failed on a conflict in data/index.json (four times in ten days,
# October 2026).
- uses: actions/checkout@v6
with: {ref: main}
- uses: actions/setup-python@v6
with: {python-version: '3.15'}
- run: chmod +x ./bash/json.sh && ./bash/json.sh
env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'}
env:
STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'
JOB_BUDGET_MINUTES: '60'
+22 -1
View File
@@ -1,19 +1,40 @@
name : Update Reviews
# Every game's review summary from IStoreBrowseService/GetItems, fifty games a
# request: about two minutes for the whole catalogue (scripts/update_reviews.py
# has the rules). It used to ask appreviews about one game at a time and, at
# 9,800 games, ran into GitHub's six-hour limit with nothing saved.
on :
schedule : [ { cron : '0 6 */2 * *' } ]
schedule : [ { cron : '0 6 * * *' } ]
workflow_dispatch :
inputs :
dry_run :
description : 'Dry run: read Steam, write nothing'
type : boolean
default : false
permissions : { contents : write }
# queue: max - a run that finds another data job running waits its turn
# (first in, first out) instead of evicting the run already waiting.
concurrency :
group : data-write
cancel-in-progress : false
queue : max
jobs :
reviews :
runs-on : ubuntu-latest
timeout-minutes : 60
steps :
-
uses : actions/checkout@v6
# main as it is when the job STARTS: a run that waited its turn must
# work on what the job before it committed.
with : { ref : main }
-
uses : actions/setup-python@v6
with : { python-version : '3.15' }
-
run : chmod +x ./bash/reviews.sh && ./bash/reviews.sh
env :
JOB_BUDGET_MINUTES : '40'
REVIEWS_DRY_RUN : ${{ inputs.dry_run }}
-2
View File
@@ -13,8 +13,6 @@ Thumbs.db
# scripts/mark_dead_games.py and explicitly staged by mark-dead-games.yml.
# It survives *.log today only because ignore rules do not apply to
# already-tracked files -- rename or recreate it and it silently vanishes.
# (scripts/dead_links.log is deliberately NOT excepted: bash/dead_links.sh
# runs `git add .`, and that log is local diagnostic output.)
!scripts/dead_games.log
*.tmp
*.temp
+48
View File
@@ -4,6 +4,54 @@ All notable changes to this awesome noob repo will be documented here.
## [Unreleased]
## [v4.4.0] – 2026-10-23 (The "Headroom" Edition)
The data pipeline grows with the catalogue again. With almost 10,000 games
and a few hundred more every day, the jobs that asked Steam about one game at
a time had run out of time; they now read Steam's store API fifty games a
request, wait their turn instead of cancelling each other, and stop in time to
save. The release also carries the tooling merged since v4.3.3 - TypeScript
7, Node 26 and Python 3.15, SonarCloud in CI, the desktop updater fix - and a
certifi bump. The desktop and Android apps stay at `2.2.1`, with no new
build. Repo public-facing version `4.3.3` → `4.4.0`.
### 🧰 Pipeline
- **The data jobs scale with the catalogue again.** With almost 10,000 games
and a few hundred more every day, three jobs that asked Steam about one
game at a time had reached GitHub's six-hour limit: on 2026-10-23 Update
Reviews was stopped at exactly six hours with nothing saved. The full
assessment, with every measurement, is `docs/plan/18-pipeline-scale.md`.
- **Reviews come from Steam's store API, for every game, every day.** Fifty
games a request, about two minutes for the catalogue, instead of every two
days and five and a half hours. The percentage is now Steam's own, which
rounds down: about 4,000 games show one point less once (Counter-Strike 2
"85%" instead of "86%"), exactly as on their store pages. A summary that
says 0 reviews is double-checked game by game before it replaces anything.
- **The daily purge needs two answers to remove a game.** Steam's store API
screens every game in a few minutes, and a game is removed only when
appdetails agrees; "unavailable" is asked twice, five minutes apart. A run
that would remove more than 100 games, or flags more than 500, stops before
writing anything. It also lists earlier removals that Steam has since
undone. It runs daily now, instead of a five-hour sweep on Mondays.
- **Check Dead Links is retired.** In its whole life it never found a dead
link: Steam sends a delisted game's page to the store front, which answers
200. The purge's screen sees delisted games (`unlisted`) instead.
- **No run is cancelled for waiting.** The data jobs share one queue that
used to hold a single waiting run: a third job cancelled it, and Update
Reviews lost five of ten runs that way. The queue now holds them all, first
in, first out, and every job checks out the latest `main` when it starts -
Auto Update JSON failed four days in ten from working on the data as it was
when the run was queued.
- **Every long job stops in time to save.** Each workflow gives its job a time
budget below its timeout: past it the job saves and commits what it did
instead of being stopped with nothing, and it warns before the budget gets
tight. The leaderboards skip the pauses meant for store pages (Top Online
~27 minutes instead of 48). The store-data and all-time-peak rotations
stretch their cycle as the catalogue grows instead of their daily runs.
- **Force Re-fetch runs in parts** (`1/8` … `8/8`, queued one after another),
and a game whose fetch fails keeps its old data.
### 🧰 Tooling
- **Desktop releases keep reaching the updater.** tauri-action v1 writes each
+93 -19
View File
@@ -222,7 +222,13 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
(tags, languages, `language_details`, `has_paid_dlc`) every 7 days,
appdetails (name, header image, description, developer/publisher, release
date, platforms, Metacritic, DRM notes) every 30. Never `appid % cycle`:
appids are mostly multiples of 10, which leaves most slices empty. It
appids are mostly multiples of 10, which leaves most slices empty. A cycle
grows with the catalogue instead of the run (`rotation.cycle_for()`: past
`PAGE_DAILY` 2,000 pages or `DETAILS_DAILY` 500 appdetails a day it becomes
14, 21, ... or 60, 90, ...) and is always a MULTIPLE of its base, so a game
keeps its weekday and its day of the month and visits only ever thin out.
The slice is visited in `rotation.daily_order()`, so a run that meets its
time budget leaves a different tail each day. It
replaces values only with NON-EMPTY ones, never writes `MANUAL_FIELDS` or
`notes`, trusts `has_paid_dlc` only from a real store page (an age gate has
no language table or tags) or a gated game's DLC prices, compares every
@@ -231,6 +237,9 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
counting as news), and bumps `last_updated` only on a real change. A game
with empty tags, languages or `language_details` is in the page group EVERY
day, first, capped at `MISSING_DAILY_CAP`; `--missing-only` runs just those.
A game whose appdetails no longer call it free is LISTED in a `::warning::`,
never written: purge removes a game only when the store API flags it too,
and this is where the opposite case shows.
A new field added to it must obey the same rules;
`scripts/tests/test_refresh_store_data.py` holds them.
- **A store page behind Steam's age gate is never parsed.** An 'Adult Only'
@@ -252,7 +261,8 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
with `l=english`: without it, a runner was once answered in Russian.
- **`all_time_peak` only ever goes UP, and `refresh_peaks.py` is what raises
it** (daily 19:30 UTC, "Refresh All-Time Peaks"). A `crc32(appid) % 30`
slice (`core/rotation.py`, shared with `refresh_store_data.py`) reads
slice (`core/rotation.py`, shared with `refresh_store_data.py`; 60, 90, ...
past `PEAK_DAILY` 400 pages a day, never shorter) reads
SteamCharts' recorded all-time peak - the only source older than our own
sampling: SteamDB forbids scraping and Valve's APIs know only today. Every
run also raises it from our samples (`peak_today`, which despite its name is
@@ -260,8 +270,9 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
`apply_players()` raises it on every sample, never from a 0. SteamCharts has
no API, terms or robots.txt, so: one page per game a month, 2-3 s apart, a
User-Agent that names the project; 500 is an app it does not track (most of
the long tail), 404 one it has never seen; a 403, 429 or challenge page
stops the run with a warning, not a failure. `refetch_all.py` must never
the long tail), 404 one it has never seen; a 403, 429 or challenge page,
or the job's time budget, stops the run with a warning, not a failure.
`refetch_all.py` must never
clear the field - it cannot be fetched again - and nothing may lower it.
The backfill is a manual run with `backfill` ticked (~4 h, start it in the
evening UTC). DISCLAIMER section 6 names the source, so rewording it reopens
@@ -293,6 +304,42 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
name and the new key are the same family, and a known label stays as it was
written (so "Easy Anti-Cheat" and "EAC" both occur; the site groups them).
`scripts/tests/test_anticheat.py` and `test_refresh_anticheat.py` hold it.
- **Reviews are Steam's own summary, read for EVERY game daily**
(`update_reviews.py`, "Update Reviews" 06:00): `GetItems` with
`include_reviews`, fifty games a request (`SteamClient.fetch_items()`, the
sweep the anti-cheat and purge jobs share), ~2 minutes. It asked
`appreviews` one game at a time until v4.4.0 and, at 9,800 games, ran into
GitHub's six-hour limit with nothing saved. Rules: the percentage is Steam's,
FLOORED (`reviews_from_summary()` floors appreviews in integers too, so an
ingest and the refresh agree; until v4.4.0 it was rounded, one point higher
on ~40% of games); a GetItems count of 0 is never written - its summary has
said 0 for a game with thousands (845070) - so a stored percentage or "N/A"
is asked on appreviews (`ASK_CAP` 300 a run, daily order) and only that
answer is written; no item, no usable summary or a failed batch keeps the
value. A record's `last_updated` moves on a change and on a HEARTBEAT: a
game read whose stamp is 14-20 days old (crc32 jitter) is stamped again.
The site calls a record older than 30 days stale (`web/src/lib/validation.ts`,
`/health`), and this job used to stamp every game on every run; the
heartbeat keeps "the pipeline confirmed this lately" true without rewriting
every record daily. `scripts/tests/test_update_reviews.py` holds it.
- **A game is purged only when two of Steam's answers agree**
(`purge_unhealthy.py`, "Purge Unhealthy Games" daily 07:00). GetItems
screens the whole catalogue (`store_items.screen_item()`: unknown to Steam,
`visible` false, `unlisted`, `release.is_coming_soon`, no `is_free`; never
`type` - a free mod is listed on purpose), and `check_game_health()`
(appdetails) must confirm each flag. An `unavailable`/404 answer is asked
again `RECHECK_AFTER` (5 min) later: appdetails answered `success:false` for
14 live games in two bursts on 2026-09-27. A failed batch or a network error
is never "gone"; blocklisted games are not asked about. More than 500 flags
or 100 confirmed removals exits non-zero BEFORE anything is written
(`--max-candidates`, `--max-removals` to override). Removals are logged to
`removed_games.jsonl` exactly as before - `reconcile.ts` reads them. A
`::notice::` names earlier removals GetItems now sees free, listed and
released (re-queue the link to bring one back: a purged game keeps its D1
decision and discovery never proposes it again). Check Dead Links is gone:
HEAD on a delisted app's page ends on the store front with 200, so in its
whole life it never found a dead link, and it took 5.5 hours.
`scripts/tests/test_purge_unhealthy.py` holds it.
- `ingest_new.py` only reads `scripts/temp_info.jsonl`, and a line is a store
LINK and nothing else: every other key is ignored and the game is built from
Steam (`merge_extension_data()` went with the archived browser extension;
@@ -305,14 +352,15 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
appdetails calls other than `type: "game"` (`status_code: not_a_game` in
`removed_games.jsonl`) - the check that makes auto-approve safe, and why a
queued link cannot add a free DLC, demo or tool. The check lives in ingest
only: in `health_checker` it would make Purge delete old records every week.
only: in `health_checker` it would make Purge delete old records every day.
- **`ingest-new.yml` is also the removal job.** `bash/ingest.sh` runs
`remove_games.py` (applies `data/blocklist.jsonl`) and then
`ingest_new.py`, and commits once ("Apply removals" when `temp_info` did not
change). A separate removal workflow would keep evicting the ingest from
`data-write`'s single pending slot, and the ingest it. The workflow checks
out `ref: main`, not the triggering SHA: a run that waited its turn would
otherwise work on the data as it was when it was queued.
change), so one commit to either file starts one run that settles both.
The workflow checks out `ref: main`, not the triggering SHA, like every
data-write workflow: a run that waited its turn would otherwise work on the
data as it was when it was queued. It has no time budget: `clear_temp()`
empties the whole queue file, so a game it had not reached would be lost.
- New games ARE discovered automatically now, by `scripts/discover_new.py`
into the D1 review queue — but discovery only PROPOSES. Publication is
decided in `/admin`: by a reviewer, batch by batch, or by the auto-approve
@@ -329,8 +377,10 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
page previews is the SQL that runs). The empty-file gate is checked again
inside the commit (`requireEmptyQueue`), so a batch never lands mid-ingest;
"no row is approved" would have stalled it for the 12 hours a skipped row
stays approved. The idle gate exists because a push while a data job waits
EVICTS that job. `shared/pipeline.ts` lists every `data-write` workflow by
stays approved. The idle gate dates from when a push while a data job waited
EVICTED that job; with `queue: max` the ingest would only queue behind it,
and the gate stays so a batch lands when its ingest can start at once.
`shared/pipeline.ts` lists every `data-write` workflow by
file, and `pipeline.test.ts` fails when a workflow joins the group without
joining the list. Settings, limits and their validation are
`shared/automation.ts`, imported by the Worker and the page. A scheduled
@@ -393,16 +443,40 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
(wrangler.jsonc, 180). `docs/PRIVACY_POLICY.md` states the number: change
both together. `ingest_queue` and `ingest_decisions` are never pruned — the
decisions are what keep a rejected game out of the discovery sweep.
- Every workflow that writes to `data/` shares `concurrency: {group: data-write}`.
A new data-writing workflow without it will race jobs that run ~2 hours.
The group holds ONE pending run - a second queueing run cancels the first -
and scheduled runs here start 2-5 hours after their cron, so a new schedule
goes where nothing long is running (the long jobs - reviews, dead links,
purge - start mid-morning UTC and run 2-4 hours; Refresh Store Data is at
16:00 for that reason).
- **Every workflow that writes to `data/` shares `concurrency: {group:
data-write, cancel-in-progress: false, queue: max}`, checks out `ref: main`,
and declares `timeout-minutes`.** `pipeline.test.ts` fails on a data-write
workflow without any of them. A new data-writing workflow outside the group
would race the others.
- `queue: max` (GitHub, 2026-05-07) keeps up to 100 runs waiting, first in,
first out. The default keeps ONE: a run queueing behind a running job
cancelled the one already waiting - Update Reviews lost five of ten runs
that way in October 2026. Mixing the two settings in one group is
undocumented, so every member has `queue: max`.
- `ref: main`: a run's commit is fixed when the run is CREATED. A scheduled
run that waited behind another data job wrote shards from before that
job's commit, and `commit_push.sh`'s rebase then conflicted on
`data/index.json` - Auto Update JSON failed four days in ten that way.
- Scheduled runs start up to an hour after their cron (2-5 hours in
September). Waiting costs nothing now, but a long new job still goes where
nothing long runs (Refresh Store Data is at 16:00 for that reason).
- **Every long loop has a time budget** (`scripts/core/budget.py`). The
workflow sets `JOB_BUDGET_MINUTES` at least 20 below its `timeout-minutes`
(`pipeline.test.ts` checks); past it a loop starts no further game, the
script saves what it did and the job commits it, and a loop that used more
than three quarters says so in a `::warning::`. Loops visit their games in
`rotation.daily_order()`, so a cut falls elsewhere the next day. A job
stopped at its timeout commits nothing and concludes `cancelled`, which
`notify-ci-failure.yml` (a reusable workflow in `poli0981/.github`) does not
report: the budget is what keeps a growing catalogue from failing silently.
`process_batch(pause=False)` for the keyed Web API (player counts); the
15-30 s rest between batches stays for store pages and appdetails. Never a
budget for `ingest_new.py`. `refetch_all.py` runs in parts (`--part K/N`)
and clears each game on a copy that replaces it only once appdetails has
answered, so a cut part leaves every other game as it was.
- **Every data job commits through `bash/commit_push.sh`.** The group
serialises workflows, not `main`: a merged PR or an /admin commit can land
during a four-hour job, and a bare `git push` is then rejected and the run
during a long job, and a bare `git push` is then rejected and the run
thrown away (purge-unhealthy, 2026-09-21). The helper rebases onto
`origin/main` and retries; none of those writers touch the shards, so the
rebase is clean. A new wrapper must use it, once per commit (Refresh Store
+11 -12
View File
@@ -5,7 +5,7 @@
[![Web App](https://img.shields.io/badge/Web%20App-Live-2563eb?style=flat&logo=svelte)](https://free-steam-games.win/)
[![Desktop](https://img.shields.io/badge/Desktop-Tauri%202-FFC131?style=flat&logo=tauri)](https://github.com/poli0981/free-steam-games-list/releases)
[![Android](https://img.shields.io/badge/Android-11%2B%20APK-3DDC84?style=flat&logo=android)](https://github.com/poli0981/free-steam-games-list/releases)
[![Health Check](https://img.shields.io/badge/Health%20Check-Weekly-orange?style=flat&logo=github-actions)](.github/workflows/purge-unhealthy.yml)
[![Health Check](https://img.shields.io/badge/Health%20Check-Daily-orange?style=flat&logo=github-actions)](.github/workflows/purge-unhealthy.yml)
![Visitors](https://visitor-badge.laobi.icu/badge?page_id=poli0981.free-steam-games-list)
[![Sponsor](https://img.shields.io/badge/Sponsor-Buy%20me%20a%20coffee-ff5f5f?style=flat&logo=buy-me-a-coffee)](.github/FUNDING.yml)
@@ -89,8 +89,8 @@ The Python pipeline that powered v2 is untouched and runs daily as before:
- Schema v2.2 (23 + manual fields per record), CC BY 4.0 (see [LICENSE-DATA](LICENSE-DATA)), sharded data in `data/data_*.jsonl` (cap 800 records/shard) plus a `data/index.json` manifest with a SHA-256 per shard.
- HTML scraper for accurate language tables, DLC pricing, user tags and the developer's anti-cheat disclosure — single GET per game.
- Health-checked: dead-link sweep every 5 days, full purge every Monday.
- Reviews-only refresh every 2 days; full refresh daily.
- Health-checked daily: Steam's store API screens every game, and a game is removed only when appdetails agrees.
- Reviews refreshed daily from Steam's own summaries; full refresh daily. Every data job queues for its turn and has a time budget, so a growing catalogue slows a refresh down instead of breaking it ([plan](docs/plan/18-pipeline-scale.md)).
</details>
@@ -151,7 +151,8 @@ scripts/
│ ├── anticheat.py # Steam's anti-cheat disclosure and the rule that applies it
│ ├── steamcharts.py # SteamCharts all-time peak reader (polite, stops on refusal)
│ ├── peaks.py # all_time_peak: only ever raised
│ ├── rotation.py # crc32 daily slices shared by the refresh jobs
│ ├── rotation.py # crc32 daily slices shared by the refresh jobs, growing with the catalogue
│ ├── budget.py # A job's time budget: stop in time to save, warn before it runs out
│ └── health_checker.py # Game status detection with cached API data
├── discover_new.py # Propose new games into the /admin review queue
├── ingest_new.py # Add queued games (health check + fetch from Steam)
@@ -160,13 +161,12 @@ scripts/
├── refresh_store_data.py # Daily rotation: tags/languages weekly, store details monthly
├── refresh_anticheat.py # Daily: every game's anti-cheat, as its developer declares it on Steam
├── refresh_peaks.py # Daily rotation: all-time peak from SteamCharts, monthly per game
├── update_reviews.py # Reviews-only (every 2 days)
├── update_reviews.py # Daily: every game's review summary from Steam, ~2 minutes
├── mark_dead_games.py # Flag online games with no players
├── check_dead_links.py # HEAD-request 404/410 scanner
├── purge_unhealthy.py # Remove delisted/not-free/invalid
├── purge_unhealthy.py # Daily: remove delisted/not-free/unreleased (store API screen + appdetails)
├── top_online.py / top_offline.py # Leaderboards
├── edit_game.py # Overrides from the command line
├── refetch_all.py # Force re-fetch ALL (manual only)
├── refetch_all.py # Force re-fetch, in parts (manual only)
└── delete_game.py # Remove games (blocklist) from the command line
```
@@ -181,14 +181,13 @@ scripts/
| Ingest New | On `scripts/temp_info.jsonl` or `data/blocklist.jsonl` push, and every 3 h | Apply removals, then add approved games |
| Top Online | Sun/Wed/Fri 03:00 UTC | Live player leaderboard |
| Top Offline | 1st + 15th of month 05:00 UTC | Leaderboard for single-player F2P concurrents |
| Update Reviews | Every 2 days 06:00 UTC | Reviews-only refresh |
| Update Reviews | Daily 06:00 UTC | Every game's review summary from Steam's store API |
| Refresh Store Data | Daily 16:00 UTC | Rotating re-fetch: tags, languages, DLC flag weekly; name, image, description, developer/publisher, release date, platforms, Metacritic, DRM monthly. Then every game's anti-cheat from Steam's disclosure (committed separately; manual run with `only: anticheat` does just that) |
| Refresh All-Time Peaks | Daily 19:30 UTC | `all_time_peak` from SteamCharts, every game once a month, plus our own samples; manual run with `backfill` asks about every game |
| Check Dead Links | Every 5 days 04:00 UTC | HEAD-request 404/410 scanner |
| Purge Unhealthy | Weekly Mon 05:00 UTC | Full health scan + removal |
| Purge Unhealthy | Daily 07:00 UTC | Store API screen of every game; removal once appdetails agrees |
| Mark Dead Games | Mon + Thu 04:30 UTC | Online + >1y + 0 players ≥14d → `is_dead=true` + 💀 note |
| Daily Snapshot | Daily 23:00 UTC | Per-game players and reviews into `data/snapshots/` |
| Force Re-fetch | Manual only | Re-fetch ALL games |
| Force Re-fetch | Manual only | Re-fetch games from scratch, one part (`1/8` … `8/8`) a run |
| Web CI / Python Tests | On `web/**` / `scripts/**` change | Type check, tests, build and output checks |
| Release Desktop / Android | On `desktop-v*` / `android-v*` tag | Build the apps into a draft release |
| Announce Discussion | On release `published` | Auto-create a GitHub Discussion in `Announcements` |
+6 -7
View File
@@ -1,13 +1,12 @@
#!/bin/bash
# Commit what a data job changed, and push it even if main moved meanwhile.
#
# A data job can run for hours (update-reviews, check-dead-links,
# purge-unhealthy), and `concurrency: data-write` serialises only the
# WORKFLOWS. main still moves underneath one: a merged pull request, an /admin
# approval, removal or override the Worker commits. A plain `git push` is then
# rejected ("fetch first")
# and the whole run is thrown away - purge-unhealthy lost four hours of work
# that way on 2026-09-21.
# A data job can run for an hour or more (refresh-store-data, the
# leaderboards), and `concurrency: data-write` serialises only the WORKFLOWS.
# main still moves underneath one: a merged pull request, an /admin approval,
# removal or override the Worker commits. A plain `git push` is then rejected
# ("fetch first") and the whole run is thrown away - purge-unhealthy lost four
# hours of work that way on 2026-09-21.
#
# None of those other writers touch the shards, so rebasing onto them and
# pushing again is safe. A genuine conflict still fails the job, loudly.
-5
View File
@@ -1,5 +0,0 @@
#!/bin/bash
set -euo pipefail
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
python scripts/check_dead_links.py
bash "$(dirname "$0")/commit_push.sh" "Dead link check [$(date +'%Y-%m-%d')]"
+4 -3
View File
@@ -3,9 +3,10 @@
#
# remove_games.py applies data/blocklist.jsonl (games the maintainer removed in
# /admin or with delete_game.py) before ingest_new.py adds the queued links.
# One workflow does both on purpose: `data-write` keeps ONE pending run and a
# newer one evicts it, so a separate removal workflow and the ingest would keep
# cancelling each other. Whichever run survives does both.
# One workflow does both: a commit to either file starts one run that settles
# the two, in that order. (It was also the only way to do it while
# `data-write` kept a single pending run and a separate removal workflow and
# the ingest kept evicting each other; the group queues them now.)
set -euo pipefail
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
python scripts/remove_games.py
+7
View File
@@ -1,5 +1,12 @@
#!/bin/bash
set -euo pipefail
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
# PURGE_DRY_RUN is the manual run's "dry run" box: screen and confirm, remove nothing.
# A guard that trips (too many flagged or confirmed) exits non-zero before
# anything is written, so set -e stops here and nothing is committed.
if [[ "${PURGE_DRY_RUN:-}" == "true" ]]; then
python scripts/purge_unhealthy.py --dry-run
exit 0
fi
python scripts/purge_unhealthy.py
bash "$(dirname "$0")/commit_push.sh" "Purge unhealthy [$(date +'%Y-%m-%d')]"
+6 -3
View File
@@ -1,7 +1,10 @@
#!/bin/bash
set -euo pipefail
echo "── Force re-fetch ALL games (v2.1 schema) ──"
# REFETCH_PART is the manual run's "part" box, K/N (scripts/refetch_all.py
# checks it): the whole catalogue no longer fits in one job.
part="${REFETCH_PART:-1/1}"
echo "── Force re-fetch, part ${part} ──"
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
python scripts/refetch_all.py
bash "$(dirname "$0")/commit_push.sh" "Force re-fetch all games → v2.1 schema [$(date +'%Y-%m-%d')]"
python scripts/refetch_all.py --part "$part"
bash "$(dirname "$0")/commit_push.sh" "Force re-fetch, part ${part} [$(date +'%Y-%m-%d')]"
+5
View File
@@ -1,5 +1,10 @@
#!/bin/bash
set -euo pipefail
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
# REVIEWS_DRY_RUN is the manual run's "dry run" box: read Steam, write nothing.
if [[ "${REVIEWS_DRY_RUN:-}" == "true" ]]; then
python scripts/update_reviews.py --dry-run
exit 0
fi
python scripts/update_reviews.py
bash "$(dirname "$0")/commit_push.sh" "Update reviews [$(date +'%Y-%m-%d')]"
+5 -4
View File
@@ -659,10 +659,11 @@ counts as *no* — the page's *Last run* line says which one stopped it:
- **The previous batch has been ingested**: `scripts/temp_info.jsonl` on `main`
is empty. Checked again inside the commit itself, so a batch never lands in
the middle of an ingest.
- **No data job is running or waiting.** The `data-write` group keeps one
waiting run, and the ingest a batch starts would cancel a job already waiting
its turn. The list of those workflows is `web/shared/pipeline.ts`, which a
test holds to the YAML.
- **No data job is running or waiting.** The `data-write` group queues its
runs (`queue: max`), so the ingest a batch starts would only wait behind such
a job; the gate makes a batch land when its ingest can start at once. The
list of those workflows is `web/shared/pipeline.ts`, which a test holds to
the YAML.
- **Today's budget is not spent.** It is reserved before the commit and given
back if the commit fails.
+170
View File
@@ -0,0 +1,170 @@
# Pipeline scale — the data workflows as the catalogue grows
## Why this note exists
On 2026-10-23 the maintainer asked for an assessment of the GitHub Actions
data workflows now that the catalogue grows by hundreds of games a day, a
direction for them, and the change. This is that assessment and the decisions
it led to. Every number below was measured on the day (`main` at 01a631bb,
9,847 games), with `gh run list` over each workflow's last 40 runs and live,
read-only requests to Steam.
The short version: three jobs asked Steam about every game one request at a
time, two seconds a game, and had reached GitHub's six-hour limit; the
`data-write` group cancelled half of the runs that waited for one of them;
and the runs that did wait worked on stale data. Steam answers fifty games a
request in its store API, and GitHub can now queue runs. The pipeline uses
both, and every long loop now stops in time to save.
## What the catalogue does to each job
The catalogue held 9,847 games and grew by about 280 a day (auto-approve is
capped at 500); Steam lists about 17,000 free games matching its filters, so
the plan is sized for 20,000.
A per-game loop cost about 2.05 s a game against store endpoints (a
1.2-2.0 s throttle, plus `process_batch`'s 15-30 s rest after every 50 games)
and about 1.0 s against the keyed Web API, of which the rests were half.
| Workflow | Before | Measured | At 20,000 games |
| --- | --- | --- | --- |
| Update Reviews (every 2 days) | `appreviews`, one game a request | 2026-10-23: stopped at exactly 6:00:00, nothing saved; 5 of the last 10 runs cancelled while waiting | impossible |
| Check Dead Links (every 5 days) | HEAD on every store page, flags 404/410 | 5.5 h on 2026-10-21; `dead_links.log` never existed and every game is `active`: Steam sends a delisted app's page to the store front, 200 | impossible |
| Purge Unhealthy (Mondays) | appdetails for every game, one answer enough to remove | 5.3 h on 2026-10-19 | impossible |
| Auto Update JSON (daily) | incomplete records only | 1 minute, but failed 4 days in 10 (below) | fine |
| Top Online (3 a week) | player count for each online game | 48 min (2,898 games) | ~1.7 h |
| Top Offline (twice a month) | player count for each offline game | 88 min (6,949 games) | ~4 h |
| Refresh Store Data (daily) | a 7-day / 30-day rotation | 65 min of a 120-minute timeout | ~2.2 h: over |
| Refresh All-Time Peaks (daily) | a 30-day rotation on SteamCharts | 14 min of a 30-minute timeout | ~29 min: at the edge |
| Force Re-fetch (manual) | everything in one run | ~6.5 s a game, 18 hours | impossible |
Three problems were not about speed:
- **A stopped job is silent.** A job killed at its timeout commits nothing and
concludes `cancelled`. `notify-ci-failure.yml` hands runs to a reusable
workflow in `poli0981/.github` that reports only `failure`, so nobody heard
that Update Reviews had stopped working.
- **The `data-write` group kept ONE pending run.** A run that queued behind a
running job cancelled the one already waiting. The ingest's three-hourly
schedule made that happen every morning: Update Reviews was cancelled on
10-05, 10-11, 10-15, 10-19 and 10-21.
- **Queued runs worked on stale data.** A run's commit is fixed when the run
is created. Auto Update JSON is created at 00:00 together with the ingest's
schedule; when the ingest ran first and committed, Auto Update JSON wrote
shards from before that commit, and `commit_push.sh`'s rebase failed on
`data/index.json` (10-15, 10-16, 10-17, 10-18). Only three workflows checked
out `ref: main`.
## What Steam's store API answers
`IStoreBrowseService/GetItems` (keyless, already used for gated games and the
anti-cheat sweep) answered for the whole catalogue in 99 requests and 105
seconds (100 ids a request; 200 also worked, 500 is refused at the edge for
the URL's length). Each item carries what the slow jobs asked for:
- `reviews.summary_filtered`: `review_count`, `percent_positive`,
`review_score_label` - the summary the store shows, in the vocabulary
`appreviews` uses ("3 user reviews", "Mixed").
- `visible`, `unlisted` (true for delisted apps), `is_free` (absent when not
free), `release.is_coming_soon`, `type`.
Compared with the stored reviews (read from `appreviews` on 10-17, rounded):
5,296 identical, 3,795 exactly one point lower - Steam floors its percentage
(85 for Counter-Strike 2, which `round()` had made 86) - 287 with a new label,
and 4 where GetItems said 0 reviews for a game the store page shows with
thousands (845070) or a few. Its 0 is not to be trusted alone.
Screening every game the way purge would: 3 `unlisted` (appdetails calls them
not free: removed), 5 without `is_free` (appdetails calls two of them free:
kept), nothing hidden, unknown or coming soon. The dry run of the new purge on
the real catalogue matched: 8 flagged, 3 removed, 5 kept by appdetails.
Of the 142 games removed before, GetItems now sees 27 as free and listed
again. Most were never really gone: 14 were rejected by the ingest in two
bursts on 2026-09-27, when appdetails answered `success:false` for live games,
and were approved again later. Eight are still out; the new purge names them.
GitHub Actions has supported `concurrency.queue: max` since 2026-05-07: up to
100 waiting runs, first in, first out, with `cancel-in-progress: false`.
## The direction
1. **Queue, don't evict.** Every data-write workflow has `queue: max` and
checks out `ref: main`, so a run that waits loses nothing and works on what
the run before it committed.
2. **Batch, don't crawl.** Reviews and the purge screen come from GetItems:
about 200 requests for the catalogue, whatever its size within reason.
Requests per game are kept for confirming a removal and for the rare
review count of 0.
3. **Bound every run.** A job sets `JOB_BUDGET_MINUTES` below its timeout;
a loop past it starts no further game and the script saves what it did.
Loops visit games in an order that changes daily, rotations grow their
cycle (a multiple of the base) instead of their daily slice, and a
`::warning::` says when a loop used more than three quarters of its budget.
4. **Drop what does nothing.** Check Dead Links goes, and so do the rests
between batches of the keyed Web API.
Considered and left: a matrix of runners per job (still a request per game,
parts racing each other's pushes, and SonarCloud's S8233 on workflow-level
write permissions with more than one job), a self-hosted runner, the Worker's
cron (the Worker may never write a shard), and a second opinion from GetItems
for the ingest's rejections (a skipped link and a rejected one both end as a
`failed` queue row; only the public log line differs).
## What changed
| Job | Now | Time at 9,847 | At 20,000 |
| --- | --- | --- | --- |
| Update Reviews | daily; GetItems, `appreviews` only for a 0 (at most 300) | ~2-5 min | ~5 min |
| Purge Unhealthy | daily; GetItems screen, appdetails confirms each flag | ~3-8 min | ~5-10 min |
| Check Dead Links | retired | 0 | 0 |
| Top Online / Offline | no rests between batches | ~27 / ~64 min | ~55 min / ~2.1 h |
| Refresh Store Data | cycles 7 → 14 past 2,000 pages a day, 30 → 60 past 500 appdetails | ~65 min | ~65-90 min |
| Refresh All-Time Peaks | cycle 30 → 60 past 400 pages a day | ~14 min | ~15 min |
| Force Re-fetch | parts `1/8` … `8/8`, queued | ~2.2 h a part | ~2.2 h a part (`1/16`…) |
Rules worth keeping (CLAUDE.md has them in full):
- **Reviews.** Steam's floored percentage, in integers (`29/100*100` is 28.99…
in floating point). A GetItems count of 0 never writes; a stored percentage
or "N/A" is asked on `appreviews` instead. No answer keeps the value.
`last_updated` moves on a change, and on a heartbeat every 14-20 days for a
game that was read: the site's "stale" badge (30 days) used to be kept quiet
by this job stamping every game every run, which also rewrote every shard.
- **Purge.** Remove only when GetItems flags a game AND `check_game_health()`
agrees; ask an `unavailable` again five minutes later; never treat a failed
batch or a network error as "gone"; stop before writing anything past 500
flags or 100 removals; log as before; name earlier removals Steam has since
undone. A purged game keeps its D1 decision, so discovery never proposes it
again - the notice is the way back.
- **Budgets.** Never for the ingest: `clear_temp()` empties its whole queue.
Refetch clears a game on a copy and swaps it in only once appdetails has
answered, oldest first, so a part cut short resumes where it stopped.
`web/shared/pipeline.test.ts` now fails on a data-write workflow without
`queue: max`, `cancel-in-progress: false`, `ref: main` on every checkout, a
`timeout-minutes`, or a budget less than 20 minutes below it.
## Load, before and after
| | Before | After |
| --- | --- | --- |
| Time the data-write group is busy | ~46 h a week | ~12 h a week |
| Requests to Steam | ~82,000 a week | ~25,000 a week |
| Runs cancelled while waiting | about one a day | none |
## Left for later
- **A timed-out run is still silent.** The reusable notifier in
`poli0981/.github` reports `failure` only; reporting `cancelled` for the
data workflows (now that waiting no longer cancels anything) belongs there.
- **Workers Builds deploys on data commits** (`docs/DEPLOYMENT.md`, "Which
commits deploy"). Daily reviews and purges add about one deploy a day until
the watch paths are set.
- **`data/snapshots/`** grows by 1.9 MB a day and nothing in the repository
reads it. A leaner format (no name, no zero rows) would quarter it.
- **The keyed player counts** are the last per-game loop. At 20,000 games
Top Offline needs about two hours; past that, sample games with no players
on a rotation instead of every run.
- **Re-queue what Steam has undone**: the purge's notice lists the removed
games that are free and on the store again.
-44
View File
@@ -1,44 +0,0 @@
#!/usr/bin/env python3
"""Dead link checker – HEAD requests only."""
import sys, os, random, time
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from datetime import datetime, timezone
from core.constants import DEAD_LINKS_LOG, BATCH_SIZE, BATCH_PAUSE_MIN, BATCH_PAUSE_MAX
from core.data_store import load_main, save_main, extract_appid, now_iso, append_note_idempotent
from core.steam_client import get_client
def main():
games = load_main()
active = [g for g in games if g.get("status", "active") == "active"]
print(f"Checking {len(active)} games...\n")
client, dead, total = get_client(), [], len(active)
for bs in range(0, total, BATCH_SIZE):
be = min(bs + BATCH_SIZE, total)
print(f"── Batch {bs//BATCH_SIZE+1} ──")
for i in range(bs, be):
g = active[i]
appid = extract_appid(g.get("link", ""))
if not appid: continue
name = g.get("name", appid)
code = client.check_store_page(appid)
if code in (404, 410):
print(f" [{i+1}/{total}] 💀 {name} ({code})")
g["status"] = "delisted"
append_note_idempotent(g, "💀 Delisted", f"💀 Delisted ({code})")
g["last_updated"] = now_iso()
dead.append({"name": name, "appid": appid, "code": code})
else:
print(f" [{i+1}/{total}] {'✓' if code==200 else f'⚠{code}'} {name[:50]}")
if be < total:
time.sleep(random.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX))
save_main(games)
if dead:
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
with open(DEAD_LINKS_LOG, "a", encoding="utf-8") as f:
f.write(f"\n── {ts} ──\n")
for d in dead: f.write(f" [{d['code']}] {d['name']} ({d['appid']})\n")
print(f"\n💀 {len(dead)} dead")
else:
print(f"\n✓ All {total} alive!")
if __name__ == "__main__": main()
+81
View File
@@ -0,0 +1,81 @@
"""
How long a data job may keep starting work, so that a catalogue which outgrows
its schedule makes a run partial instead of making it fail.
A GitHub-hosted job is stopped at its timeout-minutes (six hours at most), and
a stopped job commits nothing: Update Reviews ran for exactly six hours on
2026-10-23 and threw away every review it had read. Each data workflow sets
JOB_BUDGET_MINUTES at least twenty minutes below its timeout. A loop that
reaches the budget starts no further game, the script saves what it did, and
the job commits it. The games it did not reach keep their values until the
next run, and since the loops visit their games in an order that changes daily
(core/rotation.daily_order) the next cut falls elsewhere.
A loop that ran out, or used more than three quarters of the budget, says so
with a ::warning:: annotation on the run: the sign that the catalogue is
outgrowing the job comes before any run is lost.
JOB_BUDGET_MINUTES unset, empty, zero or not a number means no deadline, so a
run on a laptop works as it always did.
"""
import math
import os
import time
from typing import Optional
ENV = "JOB_BUDGET_MINUTES"
WARN_SHARE = 0.75
class Deadline:
"""A budget of minutes counted from when it was made: from the start of
the script, so loading and saving the catalogue count too."""
__slots__ = ("start", "seconds")
def __init__(self, minutes: float, now: Optional[float] = None):
self.start = time.monotonic() if now is None else now
self.seconds = minutes * 60
@classmethod
def from_env(cls) -> Optional["Deadline"]:
try:
minutes = float(os.environ.get(ENV, "").strip())
except ValueError:
return None
if not math.isfinite(minutes) or minutes <= 0:
return None
return cls(minutes)
@property
def minutes(self) -> float:
return self.seconds / 60
def used(self) -> float:
"""The share of the budget spent so far, 0.0 to 1.0 and beyond."""
return (time.monotonic() - self.start) / self.seconds
def expired(self) -> bool:
return self.used() >= 1.0
def expired(deadline: Optional[Deadline]) -> bool:
return deadline is not None and deadline.expired()
def report(desc: str, done: int, total: int, deadline: Optional[Deadline], log=print) -> None:
"""One line on how a loop used the budget. A run that stopped short, or
came close, gets a ::warning:: - it is how a growing catalogue shows up in
the run's annotations before it costs one."""
if deadline is None:
return
spent = deadline.used() * deadline.minutes
if done < total:
log(f"::warning::{desc}: stopped after {done} of {total} when the job's "
f"{deadline.minutes:.0f}-minute budget ran out; the other {total - done} keep "
f"their values until the next run.")
elif deadline.used() > WARN_SHARE:
log(f"::warning::{desc}: {done} done in {spent:.0f} of the job's {deadline.minutes:.0f} "
f"budgeted minutes ({deadline.used():.0%}). The catalogue is outgrowing this "
f"schedule: give the job a longer cycle or a larger budget.")
else:
log(f"{desc}: {done} done in {spent:.0f} of {deadline.minutes:.0f} budgeted minutes.")
-1
View File
@@ -7,7 +7,6 @@ import os
DATA_DIR = "data"
LEGACY_JSONL = "scripts/data.jsonl" # fallback pre-migration
TEMP_JSONL = "scripts/temp_info.jsonl"
DEAD_LINKS_LOG = "scripts/dead_links.log"
REMOVED_JSONL = "scripts/removed_games.jsonl"
GAMES_DIR = "games"
OVERRIDES_DIR = "data/overrides" # one <appid>.json per human-edited game
+1 -1
View File
@@ -442,7 +442,7 @@ _SKELETON_TEMPLATE = {
def append_note_idempotent(game: dict, marker: str, full_note: str) -> bool:
"""Append `full_note` to game['notes'] only if `marker` isn't already present.
Used by check_dead_links and mark_dead_games to avoid duplicating system
Used by mark_dead_games to avoid duplicating system
notes when a workflow re-runs over an already-marked game. Returns True
when the note was actually changed.
"""
+57 -30
View File
@@ -12,6 +12,7 @@ import time
from typing import Optional
from .anticheat import apply_anti_cheat, from_categories
from .budget import Deadline, report
from .constants import (
STEAM_API_KEY, SKIP_GENRE_TAGS,
BATCH_SIZE, BATCH_PAUSE_MIN, BATCH_PAUSE_MAX,
@@ -21,6 +22,7 @@ from .data_store import extract_appid, now_iso, is_info_complete, is_empty
from .normalize import decode_entities, normalize_release_date
from .overrides import OverrideGuard
from .peaks import raise_peak
from .rotation import daily_order, day_number
from .scraper import scrape_store_page
from .store_items import paid_dlc_from_items, parse_store_item
@@ -139,15 +141,27 @@ def apply_details(game: dict, data: dict) -> dict:
return game
def reviews_text(total: int, pct: int, label: str) -> str:
"""The stored shape of a review summary: "85% (Very Positive)"."""
if total <= 0:
return "No reviews"
return f"{pct}% ({label})" if label else f"{pct}%"
def reviews_from_summary(summary: dict) -> str:
"""appreviews' query_summary as stored. The percentage is FLOORED, as
Steam floors its own (GetItems answered 85 for Counter-Strike 2 where
round() had given 86), so the daily refresh from GetItems
(update_reviews.py) and a game's first fetch here agree. In integers:
in floating point 29/100*100 is 28.999... and floors to 28."""
total = int(summary.get("total_reviews") or 0)
pos = int(summary.get("total_positive") or 0)
pct = pos * 100 // total if total > 0 else 0
return reviews_text(total, pct, summary.get("review_score_desc", ""))
def apply_reviews(game: dict, summary: dict) -> dict:
total = summary.get("total_reviews", 0)
pos = summary.get("total_positive", 0)
if total > 0:
pct = round((pos / total) * 100)
label = summary.get("review_score_desc", "")
game["reviews"] = f"{pct}% ({label})" if label else f"{pct}%"
else:
game["reviews"] = "No reviews"
game["reviews"] = reviews_from_summary(summary)
return game
@@ -307,26 +321,44 @@ def fetch_full(game: dict, client=None, fetch_players=True,
# ──────────── Batch processor ────────────
def process_batch(games: list[dict], fn, desc="Processing"):
"""Run fn on each game with batch pauses."""
def process_batch(games: list[dict], fn, desc="Processing", pause: bool = True,
deadline: Optional[Deadline] = None) -> int:
"""Run fn on each game, BATCH_SIZE at a time. Returns how many it ran on.
pause: the 15-30 s rest between batches that keeps the store-page and
appdetails loops polite. The keyed Web API (player counts) has no
need of it: it was half the time top-online took.
deadline: a core.budget.Deadline. Once it has passed no further game is
started - the one in hand finishes - and the caller saves what was
done (core/budget.py). Never pass one where an unvisited game
would be lost, as in ingest_new.py, which empties its whole queue.
"""
total = len(games)
done = 0
for bs in range(0, total, BATCH_SIZE):
if deadline is not None and deadline.expired():
break
be = min(bs + BATCH_SIZE, total)
batch_num = bs // BATCH_SIZE + 1
total_batches = (total + BATCH_SIZE - 1) // BATCH_SIZE
print(f"\n── Batch {batch_num}/{total_batches} ({be - bs} games) ──")
for i in range(bs, be):
if deadline is not None and deadline.expired():
break
name = games[i].get("name") or games[i].get("link", "?")
print(f" [{i+1}/{total}] {desc}: {name[:50]}...", end=" ")
fn(games[i])
done += 1
print("✓")
if be < total:
pause = random.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX)
print(f" ⏳ {pause:.0f}s...")
time.sleep(pause)
if pause and done == be and be < total:
wait = random.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX)
print(f" ⏳ {wait:.0f}s...")
time.sleep(wait)
report(desc, done, total, deadline)
return done
def update_all_full(games, force=False):
def update_all_full(games, force=False, deadline: Optional[Deadline] = None):
client = get_client()
to_fetch = games if force else [g for g in games if not is_info_complete(g)]
print(f"Full update: {len(to_fetch)} to fetch, {len(games)-len(to_fetch)} skipped")
@@ -334,22 +366,12 @@ def update_all_full(games, force=False):
print("All up to date!")
return
guard = OverrideGuard.load()
process_batch(to_fetch, lambda g: fetch_full(g, client=client, guard=guard), "Fetching")
process_batch(daily_order(to_fetch, day_number()),
lambda g: fetch_full(g, client=client, guard=guard), "Fetching", deadline=deadline)
def update_reviews_only(games):
client = get_client()
print(f"Reviews update: {len(games)} games")
def fn(g):
appid = extract_appid(g.get("link", ""))
if not appid: return
s = client.fetch_reviews(appid)
if s: apply_reviews(g, s)
g["last_updated"] = now_iso()
process_batch(games, fn, "Reviews")
def update_players_only(games, type_filter: str = "online"):
def update_players_only(games, type_filter: str = "online",
deadline: Optional[Deadline] = None, day: Optional[int] = None):
"""Refresh `current_players` + `peak_today` for the games matching
`type_filter`. Default 'online' preserves the historical behaviour
used by top_online.py.
@@ -360,6 +382,10 @@ def update_players_only(games, type_filter: str = "online"):
(single-player, story, etc). Empty type_game is
excluded — those are unclassified, not offline.
- "all" → every game with a non-empty type_game.
The keyed Web API is asked without process_batch's pauses, in an order
that changes daily, so a run that meets its deadline leaves a different
tail unsampled each time.
"""
if not STEAM_API_KEY:
print("No STEAM_API_KEY.")
@@ -380,4 +406,5 @@ def update_players_only(games, type_filter: str = "online"):
if not appid: return
count = client.fetch_player_count(appid, STEAM_API_KEY)
if count is not None: apply_players(g, count)
process_batch(target, fn, "Players")
target = daily_order(target, day_number() if day is None else day)
process_batch(target, fn, "Players", pause=False, deadline=deadline)
+7 -5
View File
@@ -44,6 +44,12 @@ class HealthResult:
def reason(self): return REASON_LABELS.get(self.status, self.status)
def free_per_appdetails(data: dict) -> bool:
"""Whether appdetails calls a game free: is_free, or a price that starts at 0."""
price = data.get("price_overview") or {}
return bool(data.get("is_free", False)) or (bool(price) and price.get("initial", 1) == 0)
def check_game_health(link: str, name: str = "",
client: Optional[SteamClient] = None) -> HealthResult:
"""
@@ -70,11 +76,7 @@ def check_game_health(link: str, name: str = "",
# Reject unreleased games – an F2P tracker only lists playable titles.
if data.get("release_date", {}).get("coming_soon", False):
return HealthResult(COMING_SOON, appid, resolved, link)
is_free = data.get("is_free", False)
price = data.get("price_overview", {})
if price:
is_free = is_free or price.get("initial", 1) == 0
if not is_free:
if not free_per_appdetails(data):
return HealthResult(NOT_FREE, appid, resolved, link)
# Healthy – return data so caller can reuse it
return HealthResult(OK, appid, resolved, link, data=data)
+1 -1
View File
@@ -97,7 +97,7 @@ def preserve_machine_notes(current: str, human: str) -> str:
"""Re-attach machine-appended markers that the human value would drop.
`notes` is a hybrid field: the pipeline appends its own segments to it —
check_dead_links adds "Delisted", mark_dead_games adds "Dead game",
the retired dead-link check added "Delisted", mark_dead_games adds "Dead game",
fetcher adds "No longer free!". Those appends are one-way. mark_dead_games
only calls append_note_idempotent() when `is_dead` is still False, so once
a game is marked, a stripped marker is NEVER re-appended. A plain
+26
View File
@@ -5,7 +5,13 @@ day of a cycle, so a daily run touches 1/cycle of the catalogue.
A game's slice is crc32(appid) % cycle, NOT appid % cycle: Steam appids are
overwhelmingly multiples of 10, so `% 30` would leave 27 of the 30 monthly
slices empty and pile the whole catalogue into the other three.
The cycle grows with the catalogue (cycle_for), so a daily run stays the same
size however many games there are; and a run visits its games in an order
that changes daily (daily_order), so one that stops at its time budget
(core/budget.py) leaves a different tail each day.
"""
import math
import zlib
from datetime import date, datetime, timezone
from typing import Optional
@@ -13,6 +19,26 @@ from typing import Optional
from .data_store import extract_appid
def cycle_for(count: int, base: int, daily: int) -> int:
"""The length of a rotation over `count` games: `base` days while a daily
slice stays within `daily` games, then the smallest MULTIPLE of `base`
that keeps it there.
A multiple, because crc32 % (k * base) leaves crc32 % base unchanged: a
game stays on the same day of the base cycle, so when the cycle grows its
visits only thin out - never two within `base` days. SteamCharts keeps its
one page per game a month, and the store page its weekday.
"""
if count <= 0 or daily <= 0:
return base
return base * max(1, math.ceil(count / (base * daily)))
def daily_order(games: list[dict], day: int) -> list[dict]:
"""The games in an order that is fixed within a day and changes the next."""
return sorted(games, key=lambda g: zlib.crc32(f"{day}:{extract_appid(g.get('link', ''))}".encode()))
def slice_of(appid: str, cycle: int) -> int:
"""The day of the cycle on which this game is refreshed. Stable forever:
crc32 is fixed, so a game keeps its weekday and its day of the month."""
+17 -6
View File
@@ -259,12 +259,16 @@ class SteamClient:
out[str(item["id"])] = item
return out
def fetch_anticheat_items(self, appids, give_up_after: int = 10) -> tuple[dict[str, dict], set[str]]:
"""GetItems asking only for extra_details - each game's anti-cheat
disclosure - for a whole catalogue: ({appid: item}, ids not answered).
def fetch_items(self, appids, data_request: dict,
give_up_after: int = 10) -> tuple[dict[str, dict], set[str]]:
"""GetItems for a whole catalogue, a batch at a time: ({appid: item},
ids not answered). This is how the daily sweeps read every game in a
hundred-odd requests instead of one request per game.
Unlike fetch_store_items() a failed batch costs only its own ids, which
the caller must treat as unknown, never as "no disclosure". After
the caller must treat as unknown - never as "no reviews", "no
disclosure" or "gone from the store". An id that IS answered for but is
missing from `items` is one Steam does not know (success 15). After
`give_up_after` failed batches in a row the rest is not asked at all.
"""
ids = [str(a) for a in appids if str(a).isdigit()]
@@ -276,7 +280,7 @@ class SteamClient:
if in_a_row >= give_up_after:
failed.update(chunk)
continue
items = self._store_items_batch(chunk, {"include_extra_details": True})
items = self._store_items_batch(chunk, data_request)
if items is None:
failed.update(chunk)
in_a_row += 1
@@ -286,6 +290,11 @@ class SteamClient:
out[str(item["id"])] = item
return out, failed
def fetch_anticheat_items(self, appids, give_up_after: int = 10) -> tuple[dict[str, dict], set[str]]:
"""fetch_items() asking only for extra_details - each game's anti-cheat
disclosure (core/anticheat.py)."""
return self.fetch_items(appids, {"include_extra_details": True}, give_up_after)
def fetch_tag_names(self) -> dict[int, str]:
"""{tagid: English name} from IStoreService/GetTagList, fetched once per
client. A failure is not cached, and returns {} (no tag is named)."""
@@ -343,7 +352,9 @@ class SteamClient:
return body if body.get("success") else None
def check_store_page(self, appid: str) -> int:
"""HEAD request only – lightweight dead link check."""
"""HEAD the store page: check_game_health()'s last resort when
appdetails answers without data. A delisted app redirects to the store
front and ends on 200, so this cannot tell one from a live game."""
self._throttle_store()
try:
resp = self._session.head(
+60
View File
@@ -16,6 +16,10 @@ Checked against stored HTML scrapes of 40 random games: 37 identical tag lists
(the other three had drifted since they were scraped), 40 identical language
lists, no unknown tag id. Tag ids are named by IStoreService/GetTagList.
The daily sweeps read the same items for the whole catalogue, a hundred-odd
requests where a request per game took hours: review_summary() for
update_reviews.py, screen_item() for purge_unhealthy.py.
Everything here is pure; steam_client.py makes the requests.
"""
from typing import Iterable, Optional
@@ -120,6 +124,62 @@ def parse_store_item(item: dict, tag_names: dict[int, str]) -> dict:
}
def review_summary(item: Optional[dict]) -> Optional[tuple[int, int, str]]:
"""(review_count, percent_positive, label) from an item asked for with
include_reviews - Steam's own summary, the one its store shows - or None
when the item says nothing usable: no item, no summary, a count without a
percentage. A count of 0 comes back as (0, 0, ""), and is NOT to be
trusted on its own: the summary has said 0 for a game with thousands of
reviews (845070, 2026-10-23), and lags behind a new game's first ones.
percent_positive is floored, as on the store, where round() would round
up (85 for Counter-Strike 2, which round() had made 86).
"""
summary = ((item or {}).get("reviews") or {}).get("summary_filtered")
if not isinstance(summary, dict):
return None
count = _int(summary.get("review_count"))
if count < 0:
return None
if count == 0:
return (0, 0, "")
pct = _int(summary.get("percent_positive"))
if not 0 <= pct <= 100:
return None
return (count, pct, str(summary.get("review_score_label") or "").strip())
# Why Steam's store says a game may no longer belong in the catalogue
# (screen_item), for purge_unhealthy.py to confirm with appdetails.
UNKNOWN = "unknown" # Steam does not know the app (success 15)
HIDDEN = "hidden" # visible: false
UNLISTED = "unlisted" # taken off the store
COMING_SOON = "coming_soon" # not released (any more, or yet)
NOT_FREE = "not_free" # no is_free flag
def screen_item(item: Optional[dict]) -> Optional[str]:
"""Why an item asked for with include_release flags its game, or None
for a game that is on the store, released and free. `item` None means
Steam answered the batch without it: an app it does not know.
Only a flag, never a verdict: purge_unhealthy.py removes a game only
when appdetails agrees. `type` is not read - a free mod is in the
catalogue on purpose (Black Reliquary); that check belongs to ingest.
"""
if item is None:
return UNKNOWN
if item.get("visible") is False:
return HIDDEN
if item.get("unlisted"):
return UNLISTED
if (item.get("release") or {}).get("is_coming_soon"):
return COMING_SOON
if not item.get("is_free"):
return NOT_FREE
return None
def paid_dlc_from_items(items: Iterable[dict]) -> Optional[bool]:
"""Whether any of these DLC items costs money. None when there is no item to
judge by (every DLC hidden, or the request failed): unknown, not "free".
+205 -41
View File
@@ -1,48 +1,212 @@
#!/usr/bin/env python3
"""Purge unhealthy games."""
import sys, os, random, time
"""
Take out the games that left the store, stopped being free or are not out yet. Daily.
A game is removed only when two of Steam's answers agree:
1. the screen - IStoreBrowseService/GetItems for the whole catalogue, a
hundred-odd requests (core/store_items.screen_item): Steam does not know
the app, it is hidden or unlisted, coming soon, or has no is_free flag;
2. the confirmation - appdetails for each flagged game, through the same
check_game_health() the ingest uses. An "unavailable" (or 404) answer is
asked again RECHECK_AFTER seconds later: appdetails has answered
success:false for live games in bursts (14 of them on 2026-09-27).
This used to ask appdetails about every game, one at a time: 5.3 hours at
9,800 games, with one answer enough to remove one.
The rules:
- A batch GetItems did not answer flags nothing, and neither does a network
error at the confirmation: unknown is never "gone".
- Blocklisted games are left to save_main(), which drops them anyway, and
are not asked about.
- More than --max-candidates flagged, or more than --max-removals confirmed,
stops the run with an error BEFORE anything is written: that is a broken
answer from Steam, not a wave of delistings.
- Removals are logged to scripts/removed_games.jsonl as they always were
(reason and status_code from core/health_checker.py): the Worker's
reconcile reads that file.
- Games removed in the past that GetItems now sees free, listed and released
are listed in a ::notice::, so a removal Steam has since undone can be
brought back by queueing its link again.
Usage (from the repository root):
python scripts/purge_unhealthy.py
python scripts/purge_unhealthy.py --dry-run
python scripts/purge_unhealthy.py --max-removals 300 # a deliberate mass removal
"""
import argparse
import os
import sys
import time
from collections import Counter
from typing import Optional
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.constants import REMOVED_JSONL, BATCH_SIZE, BATCH_PAUSE_MIN, BATCH_PAUSE_MAX
from core.data_store import load_main, save_main, load_jsonl, save_jsonl, dedup_removed, now_iso
from core.blocklist import blocked_appids
from core.budget import Deadline, expired
from core.constants import REMOVED_JSONL
from core.data_store import (
dedup_removed, extract_appid, load_jsonl, load_main, now_iso, save_jsonl, save_main,
)
from core.health_checker import (
COMING_SOON, NOT_FOUND_404, NOT_FOUND_410, NOT_FREE, UNAVAILABLE, check_game_health,
)
from core.steam_client import get_client
from core.health_checker import check_game_health
from core.store_items import screen_item
def main():
MAX_CANDIDATES = 500
MAX_REMOVALS = 100
RECHECK_AFTER = 300 # seconds
SCREEN_REQUEST = {"include_release": True}
# appdetails answers worth asking twice: the ones a bad moment produces.
RECHECKED = frozenset({UNAVAILABLE, NOT_FOUND_404, NOT_FOUND_410})
# Removal reasons a game can come back from: what Steam says, not a person.
RECOVERABLE = frozenset({UNAVAILABLE, NOT_FOUND_404, NOT_FOUND_410, NOT_FREE, COMING_SOON})
def screen(games: list[dict], items: dict, failed: set, blocked: set) -> list[tuple[dict, str]]:
"""The games Steam's store flags, with why. A game without an appid is
flagged too: check_game_health() calls its link invalid without a request."""
out = []
for game in games:
appid = extract_appid(game.get("link", ""))
if not appid:
out.append((game, "invalid"))
elif appid not in blocked and appid not in failed:
why = screen_item(items.get(appid))
if why:
out.append((game, why))
return out
def confirm(candidates: list[tuple[dict, str]], client, deadline: Optional[Deadline] = None,
log=print) -> tuple[list, Counter]:
"""Ask appdetails about each flagged game. Returns ([(game, HealthResult)]
to remove, tally). An unavailable answer is asked again RECHECK_AFTER
seconds after the first ask; past the deadline nothing more is asked and
the rest stay."""
removals: list = []
recheck: list[dict] = []
tally: Counter = Counter()
first_ask = None
def judge(game: dict, result) -> None:
if result.should_remove:
removals.append((game, result))
elif result.is_healthy:
tally["kept"] += 1
log(f" ✓ kept {result.appid} {result.name[:40]}: appdetails says it is healthy")
else:
tally["network"] += 1
for game, why in candidates:
if expired(deadline):
tally["unasked"] += 1
continue
first_ask = time.monotonic() if first_ask is None else first_ask
result = check_game_health(game.get("link", ""), game.get("name", ""), client)
if result.status in RECHECKED:
recheck.append(game)
else:
judge(game, result)
if recheck:
wait = RECHECK_AFTER - (time.monotonic() - first_ask)
if wait > 0:
log(f" {len(recheck)} answered as unavailable; asking again in {wait:.0f} s")
time.sleep(wait)
for game in recheck:
if expired(deadline):
tally["unasked"] += 1
continue
judge(game, check_game_health(game.get("link", ""), game.get("name", ""), client))
return removals, tally
def recoverable(removed_rows: list[dict], live: set, blocked: set, client) -> list[str]:
"""Earlier removals Steam has since undone: "appid name" for each game
removed for a store reason that GetItems now sees free, listed and
released, and that is neither back in the catalogue nor blocklisted."""
names: dict[str, str] = {}
for row in removed_rows:
appid = str(row.get("appid") or extract_appid(row.get("link", "")) or "")
if appid and row.get("status_code") in RECOVERABLE and appid not in live | blocked:
names[appid] = (row.get("name") or "").strip()
if not names:
return []
items, failed = client.fetch_items(list(names), SCREEN_REQUEST)
return [f"{a} {names[a]}".strip() for a in names
if a not in failed and a in items and screen_item(items[a]) is None]
def main(argv: Optional[list[str]] = None) -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0].strip(),
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--dry-run", action="store_true", help="screen and confirm, write nothing")
ap.add_argument("--max-candidates", type=int, default=MAX_CANDIDATES,
help=f"stop when the screen flags more games than this (default {MAX_CANDIDATES})")
ap.add_argument("--max-removals", type=int, default=MAX_REMOVALS,
help=f"stop when more games than this are confirmed (default {MAX_REMOVALS})")
args = ap.parse_args(argv)
deadline = Deadline.from_env()
games = load_main()
if not games: print("Empty."); return
print(f"Scanning {len(games)} games...\n")
if not games:
print("Empty.")
return 0
client = get_client()
remove_set, log = set(), []
stats = {"ok": 0, "rm": 0, "net": 0}
total = len(games)
for bs in range(0, total, BATCH_SIZE):
be = min(bs + BATCH_SIZE, total)
print(f"── Batch {bs//BATCH_SIZE+1} ──")
for i in range(bs, be):
g = games[i]
name = g.get("name") or g.get("link", "?")
r = check_game_health(g.get("link",""), name, client)
if r.is_healthy:
stats["ok"] += 1
elif r.should_remove:
print(f" [{i+1}/{total}] ❌ {name[:50]} → {r.reason}")
remove_set.add(i)
log.append({"link":g.get("link",""),"name":r.name or name,
"appid":r.appid or "","reason":r.reason,
"status_code":r.status,"removed_at":now_iso()})
stats["rm"] += 1
else:
stats["net"] += 1
if be < total:
time.sleep(random.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX))
if remove_set:
cleaned = [g for i,g in enumerate(games) if i not in remove_set]
save_main(cleaned)
existing = load_jsonl(REMOVED_JSONL)
save_jsonl(REMOVED_JSONL, dedup_removed(existing + log))
print(f"\nPurged {len(remove_set)}, remaining {len(cleaned)}")
else:
print("\nAll healthy.")
print(f"✅{stats['ok']} ❌{stats['rm']} ⚠{stats['net']}")
blocked = blocked_appids()
ids = [a for a in (extract_appid(g.get("link", "")) for g in games) if a and a not in blocked]
print(f"Screening {len(ids)} games with Steam's store API...")
items, failed = client.fetch_items(ids, SCREEN_REQUEST)
if ids and len(failed) == len(ids):
print("::warning::GetItems answered for no game: nothing screened, nothing removed.")
return 0
if failed:
print(f"::warning::GetItems did not answer for {len(failed)} games; they are screened again tomorrow.")
if __name__ == "__main__": main()
candidates = screen(games, items, failed, blocked)
flags = Counter(why for _, why in candidates)
print(f"{len(candidates)} flagged: " + (", ".join(f"{k} {v}" for k, v in flags.most_common()) or "none"))
if len(candidates) > args.max_candidates:
print(f"::error::The screen flagged {len(candidates)} games, more than --max-candidates "
f"{args.max_candidates}. That is a broken answer, not a wave of delistings: nothing "
f"was asked or written. Run with a higher --max-candidates if it is real.")
return 2
removals, tally = confirm(candidates, client, deadline)
for game, result in removals:
print(f" ❌ {result.appid} {(result.name or game.get('name', ''))[:50]} → {result.reason}")
print(f"Confirmed {len(removals)}; kept {tally['kept']} that appdetails calls healthy, "
f"{tally['network']} unanswered, {tally['unasked']} not asked (budget)")
if len(removals) > args.max_removals:
print(f"::error::{len(removals)} removals, more than --max-removals {args.max_removals}: "
f"nothing written. Run with a higher --max-removals if they are real.")
return 2
back = recoverable(load_jsonl(REMOVED_JSONL), set(ids), blocked, client)
if back:
print(f"::notice::{len(back)} removed game(s) are free and on the store again: "
f"{'; '.join(back[:20])}{' …' if len(back) > 20 else ''}. Queue a link in "
f"scripts/temp_info.jsonl to bring one back.")
if args.dry_run:
print("Dry run: nothing written.")
return 0
if not removals:
print("All healthy.")
return 0
gone = {id(game) for game, _ in removals}
save_main([g for g in games if id(g) not in gone])
log = [{"link": g.get("link", ""), "name": r.name or g.get("name", ""), "appid": r.appid or "",
"reason": r.reason, "status_code": r.status, "removed_at": now_iso()}
for g, r in removals]
save_jsonl(REMOVED_JSONL, dedup_removed(load_jsonl(REMOVED_JSONL) + log))
print(f"Purged {len(removals)}, remaining {len(games) - len(removals)}")
return 0
if __name__ == "__main__":
sys.exit(main())
+67 -20
View File
@@ -1,15 +1,33 @@
#!/usr/bin/env python3
"""
Force re-fetch ALL games. MANUAL ONLY.
Backup → migrate → clear fetchable → re-fetch → save.
Backup → migrate → re-fetch each game from scratch → save.
At about six seconds a game the whole catalogue takes far longer than a job
may run, so it is done in parts: --part K/N takes the games whose
crc32(appid) % N is K-1 (core/rotation.py), and the workflow's runs queue
one behind another. Each game is cleared and fetched on a copy that replaces
it only once appdetails has answered, so a failed request - or the job
budget running out (core/budget.py) - leaves every other game exactly as it
was. The oldest last_updated goes first: running a part that was cut short
again carries on where it stopped.
"""
import sys, os, shutil
import argparse
import copy
import os
import re
import shutil
import sys
from collections import Counter
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.budget import Deadline
from core.constants import DATA_DIR, STEAM_API_KEY
from core.data_store import load_main, save_main, migrate_record, now_iso
from core.data_store import extract_appid, load_main, save_main, migrate_record
from core.fetcher import fetch_full, process_batch
from core.overrides import OverrideGuard
from core.rotation import slice_of
from core.steam_client import get_client
CLEARABLE = {
@@ -49,13 +67,51 @@ def clear_fetchable(game):
game["metacritic"] = "N/A"
def main():
def parse_part(text: str) -> tuple[int, int]:
"""'K/N' -> (K, N) with 1 <= K <= N."""
m = re.fullmatch(r"\s*(\d+)\s*/\s*(\d+)\s*", text or "")
if not m or not 1 <= int(m.group(1)) <= int(m.group(2)):
raise argparse.ArgumentTypeError(f"expected K/N with 1 <= K <= N, got {text!r}")
return int(m.group(1)), int(m.group(2))
def in_part(game: dict, part: tuple[int, int]) -> bool:
appid = extract_appid(game.get("link", ""))
return bool(appid) and slice_of(appid, part[1]) == part[0] - 1
def refetch_one(game: dict, client, guard) -> bool:
"""Re-fetch one game from scratch, in place, or leave it exactly as it was
when appdetails does not answer. Returns whether it was re-fetched."""
appid = extract_appid(game.get("link", ""))
if not appid:
return False
status, details = client.fetch_app_details_full(appid)
if status != "ok" or not details:
return False
fresh = copy.deepcopy(game)
clear_fetchable(fresh)
fetch_full(fresh, client=client, prefetched_details=details, guard=guard)
game.clear()
game.update(fresh)
return True
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0].strip())
ap.add_argument("--part", type=parse_part, default=(1, 1),
help="K/N: re-fetch only the K-th of N parts of the catalogue (default 1/1)")
args = ap.parse_args(argv)
deadline = Deadline.from_env()
games = load_main()
if not games:
print("Empty.")
return
total = len(games)
print(f"Force re-fetch: {total} games\n")
targets = [g for g in games if in_part(g, args.part)]
# Oldest first: a part cut short carries on where it stopped next time.
targets.sort(key=lambda g: g.get("last_updated") or "")
print(f"Force re-fetch: part {args.part[0]}/{args.part[1]}, {len(targets)} of {len(games)} games\n")
backup_dir = DATA_DIR + ".bak"
if os.path.isdir(backup_dir):
@@ -63,30 +119,21 @@ def main():
shutil.copytree(DATA_DIR, backup_dir)
print(f"Backup -> {backup_dir}")
# Migrate + clear
for g in games:
migrate_record(g)
clear_fetchable(g)
print(f"Cleared fetchable fields. Re-fetching...\n")
if not STEAM_API_KEY:
print("⚠ No STEAM_API_KEY – players skipped")
client = get_client()
guard = OverrideGuard.load()
process_batch(games, lambda g: fetch_full(g, client=client, guard=guard), "Re-fetching")
stamp = now_iso()
for g in games:
g["last_updated"] = stamp
tally: Counter = Counter()
process_batch(targets, lambda g: tally.update([refetch_one(g, client, guard)]),
"Re-fetching", deadline=deadline)
save_main(games)
# Stats
has = lambda k: sum(1 for g in games if g.get(k) and g[k] not in ("", "N/A", [], False))
print(f"\n{'═'*50}")
print(f"Done {total} games. Backup: {backup_dir}")
print(f" name: {has('name')} | dev: {has('developer')} | pub: {has('publisher')}")
print(f" plat: {has('platforms')} | lang: {has('languages')} | tags: {has('tags')}")
print(f"Re-fetched {tally[True]} of {len(targets)} ({tally[False]} left as they were: "
f"appdetails did not answer). Backup: {backup_dir}")
if __name__ == "__main__":
+25 -9
View File
@@ -1,7 +1,8 @@
#!/usr/bin/env python3
"""
Refresh all_time_peak - the most players each game has had at once - from
SteamCharts, a thirtieth of the catalogue a day.
SteamCharts, a thirtieth of the catalogue a day (less, once that passes
PEAK_DAILY games).
core/peaks.py explains the field; core/steamcharts.py the source and why it is
read so carefully. Each run does two things:
@@ -12,7 +13,9 @@ read so carefully. Each run does two things:
field for the whole catalogue on the first run.
2. For the games whose slice is due (crc32(appid) % 30, see core/rotation.py:
every game once a month), read SteamCharts' recorded all-time peak and
raise the field to it.
raise the field to it. Past PEAK_DAILY games a day the cycle grows to
60, 90, ... days (rotation.cycle_for), never shorter: SteamCharts is still
asked about a game at most once a month.
The rules:
- all_time_peak only ever goes UP. A failed request, an untracked app or a
@@ -23,8 +26,9 @@ The rules:
never seen; both are normal and simply skipped until next month.
- A 403, a 429 or a Cloudflare challenge stops the SteamCharts part of the
run (everything already raised is kept and saved), and so do ten errors in
a row. The run still succeeds, with a warning: a refusal is SteamCharts'
call to make, not a pipeline failure.
a row, and so does the job's time budget (core/budget.py). The run still
succeeds, with a warning: a refusal is SteamCharts' call to make, not a
pipeline failure.
- Everything is written by save_main().
Usage (from the repository root):
@@ -41,12 +45,16 @@ from typing import Optional
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.budget import Deadline, expired
from core.data_store import extract_appid, load_main, migrate_record, save_main
from core.peaks import observed_peak, raise_peak
from core.rotation import day_number, due
from core.rotation import cycle_for, daily_order, day_number, due
from core.steamcharts import SteamChartsClient
PEAK_CYCLE_DAYS = 30
# The most SteamCharts pages a day before the cycle grows (rotation.cycle_for):
# about 17 minutes at 2-3 s a page.
PEAK_DAILY = 400
MAX_CONSECUTIVE_ERRORS = 10
@@ -55,13 +63,19 @@ def raise_from_samples(games: list[dict]) -> int:
return sum(1 for g in games if raise_peak(g, observed_peak(g)))
def raise_from_steamcharts(targets: list[dict], client, log=print) -> Counter:
def raise_from_steamcharts(targets: list[dict], client, log=print,
deadline: Optional[Deadline] = None) -> Counter:
"""Step 2. Returns a tally of the outcomes: "raised", "unchanged",
"untracked", "error", and "skipped" for games never asked because the run
stopped early."""
tally: Counter = Counter()
errors_in_a_row = 0
for i, game in enumerate(targets):
if expired(deadline):
tally["skipped"] += len(targets) - i
log(f"::warning::The job's time budget ran out with {len(targets) - i} SteamCharts "
f"pages left; those games are asked on their next turn.")
break
appid = extract_appid(game.get("link", ""))
if not appid:
continue
@@ -101,6 +115,7 @@ def main(argv: Optional[list[str]] = None) -> None:
ap.add_argument("--dry-run", action="store_true", help="fetch and report, write nothing")
args = ap.parse_args(argv)
deadline = Deadline.from_env()
games = load_main()
for g in games:
migrate_record(g)
@@ -109,13 +124,14 @@ def main(argv: Optional[list[str]] = None) -> None:
print(f"{len(games)} games; all_time_peak raised from our own samples for {from_samples}")
day = day_number() if args.day is None else args.day
targets = games if args.all else [g for g in games if due(g, PEAK_CYCLE_DAYS, day)]
cycle = cycle_for(len(games), PEAK_CYCLE_DAYS, PEAK_DAILY)
targets = games if args.all else [g for g in daily_order(games, day) if due(g, cycle, day)]
if args.limit is not None:
targets = targets[: args.limit]
scope = "every game" if args.all else f"slice {day % PEAK_CYCLE_DAYS}/{PEAK_CYCLE_DAYS}"
scope = "every game" if args.all else f"slice {day % cycle}/{cycle}"
print(f"SteamCharts: {len(targets)} games ({scope})")
tally = raise_from_steamcharts(targets, SteamChartsClient()) if targets else Counter()
tally = raise_from_steamcharts(targets, SteamChartsClient(), deadline=deadline) if targets else Counter()
summary = ", ".join(f"{k} {v}" for k, v in tally.most_common()) or "nothing asked"
print(f"\nSteamCharts: {summary}")
+53 -24
View File
@@ -18,8 +18,13 @@ Two groups, each on its own rotation, a slice of the catalogue per daily run:
developer, publisher, release_date, platforms,
metacritic, drm_notes every 30 days
That is ~630 store pages and ~150 appdetails calls a day - about 40 minutes
at the client's throttle - instead of 8,800 requests in one run.
That was ~1,400 store pages and ~330 appdetails calls a day at 9,800 games,
about an hour at the client's throttle. The cycles grow with the catalogue
(core/rotation.cycle_for): past PAGE_DAILY pages or DETAILS_DAILY appdetails
a day, a cycle doubles (triples, ...) instead of the run growing, and a game
keeps its weekday and its day of the month. A run that still meets its time
budget (core/budget.py) saves what it did; the slice is visited in an order
that changes daily, so the cut falls elsewhere next time.
A game's slice is crc32(appid) % cycle, NOT appid % cycle: Steam appids are
overwhelmingly multiples of 10, so `% 30` would leave 27 of the 30 monthly
@@ -37,7 +42,10 @@ The rules, because these fields were fill-if-empty until now:
gate, a delisted app or a changed page layout leaves the stored value alone.
- MANUAL_FIELDS are never written (genre, type_game, anti_cheat, safe, ...),
and neither is `notes`: health checks and "no longer free" belong to
check_dead_links.py, purge_unhealthy.py and mark_dead_games.py.
purge_unhealthy.py and mark_dead_games.py. A game appdetails no longer
calls free is only listed, in a ::warning:: - purge removes a game only
when Steam's store API flags it too, and this is where the opposite case
shows.
- has_paid_dlc is read only from a real store page - one with a language
table or tags - or, for a gated game, from the prices of its DLC. An
age-gate page has no DLC section and would read as False; when neither
@@ -60,26 +68,31 @@ Usage (from the repository root):
import argparse
import os
import sys
import zlib
from collections import Counter
from typing import Optional
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.budget import Deadline
from core.data_store import (
ELLIPSIS, extract_appid, is_empty, load_main, migrate_record, now_iso, save_main,
truncate_description,
)
from core.fetcher import process_batch, scrape_store_fields
from core.health_checker import free_per_appdetails
from core.normalize import (
clean_list, clean_text, decode_entities, normalize_records, normalize_release_date,
)
# The rotation (crc32 slices) is shared with refresh_peaks.py.
from core.rotation import day_number, due
from core.rotation import cycle_for, daily_order, day_number, due
from core.steam_client import get_client
PAGE_CYCLE_DAYS = 7
DETAILS_CYCLE_DAYS = 30
# The most a day's slice may hold before its cycle grows (rotation.cycle_for):
# with the repairs, about an hour and a half at the client's throttle.
PAGE_DAILY = 2000
DETAILS_DAILY = 500
# Games with empty store fields taken a day on top of the rotation. 25 on
# 2026-10-10; the cap is for the day something breaks for every game at once,
# so a two-hour job cannot turn into a whole-catalogue crawl.
@@ -205,8 +218,10 @@ def apply_app_details(game: dict, data: dict) -> list[str]:
return changed
def refresh_game(game: dict, client, page: bool, details: bool) -> list[str]:
"""Refresh one game in place. Returns the fields that changed."""
def refresh_game(game: dict, client, page: bool, details: bool,
not_free: Optional[list] = None) -> list[str]:
"""Refresh one game in place. Returns the fields that changed. A game
whose appdetails no longer call it free is added to `not_free`."""
appid = extract_appid(game.get("link", ""))
if not appid:
return []
@@ -216,6 +231,8 @@ def refresh_game(game: dict, client, page: bool, details: bool) -> list[str]:
data = client.fetch_app_details(appid)
if data:
changed += apply_app_details(game, data)
if not_free is not None and not free_per_appdetails(data):
not_free.append(f"{appid} {game.get('name', '')[:40]}")
if page:
# The appdetails just fetched spare the DLC check a second request.
scraped = scrape_store_fields(client, appid, details=data)
@@ -233,29 +250,31 @@ def missing_store_fields(game: dict) -> bool:
def build_plan(games: list[dict], day: int, want_page: bool = True,
want_details: bool = True, missing_only: bool = False) -> list[tuple]:
want_details: bool = True, missing_only: bool = False,
page_cycle: int = PAGE_CYCLE_DAYS,
details_cycle: int = DETAILS_CYCLE_DAYS) -> list[tuple]:
"""[(game, page, details)] for one day: the games with empty store fields
first (page group, every day), then the day's rotation slices.
The repairs are capped at MISSING_DAILY_CAP and ordered by a hash that
changes daily, so a residue Steam never fills cannot hold the same places
forever. missing_only keeps just the repairs, store page only.
Both parts come in an order that changes daily (rotation.daily_order):
the repairs are capped at MISSING_DAILY_CAP, so a residue Steam never
fills cannot hold the same places forever, and a run that meets its time
budget leaves a different part of the slice for the next one.
missing_only keeps just the repairs, store page only.
"""
repairs: list[dict] = []
if want_page:
repairs = [g for g in games if missing_store_fields(g)]
repairs.sort(key=lambda g: zlib.crc32(f"{day}:{extract_appid(g.get('link', ''))}".encode()))
repairs = repairs[:MISSING_DAILY_CAP]
repairs = daily_order([g for g in games if missing_store_fields(g)], day)[:MISSING_DAILY_CAP]
if missing_only:
return [(g, True, False) for g in repairs]
plan = [(g, True, want_details and due(g, DETAILS_CYCLE_DAYS, day)) for g in repairs]
plan = [(g, True, want_details and due(g, details_cycle, day)) for g in repairs]
taken = {id(g) for g in repairs}
for g in games:
for g in daily_order(games, day):
if id(g) in taken:
continue
page = want_page and due(g, PAGE_CYCLE_DAYS, day)
details = want_details and due(g, DETAILS_CYCLE_DAYS, day)
page = want_page and due(g, page_cycle, day)
details = want_details and due(g, details_cycle, day)
if page or details:
plan.append((g, page, details))
return plan
@@ -273,6 +292,7 @@ def main(argv: Optional[list[str]] = None) -> None:
ap.add_argument("--dry-run", action="store_true", help="fetch and report, write nothing")
args = ap.parse_args(argv)
deadline = Deadline.from_env()
day = day_number() if args.day is None else args.day
games = load_main()
for g in games:
@@ -284,7 +304,10 @@ def main(argv: Optional[list[str]] = None) -> None:
want_page = args.only in (None, "page")
want_details = args.only in (None, "details")
plan = build_plan(games, day, want_page, want_details, missing_only=args.missing_only)
page_cycle = cycle_for(len(games), PAGE_CYCLE_DAYS, PAGE_DAILY)
details_cycle = cycle_for(len(games), DETAILS_CYCLE_DAYS, DETAILS_DAILY)
plan = build_plan(games, day, want_page, want_details, missing_only=args.missing_only,
page_cycle=page_cycle, details_cycle=details_cycle)
if args.limit is not None:
plan = plan[: args.limit]
@@ -292,28 +315,34 @@ def main(argv: Optional[list[str]] = None) -> None:
n_details = sum(1 for _, _, d in plan if d)
n_missing = sum(1 for g, p, _ in plan if p and missing_store_fields(g))
print(f"{len(games)} games; day {day}: store page for {n_page} "
f"(slice {day % PAGE_CYCLE_DAYS}/{PAGE_CYCLE_DAYS}, {n_missing} with empty store fields), "
f"appdetails for {n_details} (slice {day % DETAILS_CYCLE_DAYS}/{DETAILS_CYCLE_DAYS})")
f"(slice {day % page_cycle}/{page_cycle}, {n_missing} with empty store fields), "
f"appdetails for {n_details} (slice {day % details_cycle}/{details_cycle})")
if not plan:
return
client = get_client()
flags = {id(g): (page, details) for g, page, details in plan}
fields: Counter = Counter()
not_free: list[str] = []
touched = 0
def refresh(game: dict) -> None:
nonlocal touched
page, details = flags[id(game)]
changed = refresh_game(game, client, page=page, details=details)
changed = refresh_game(game, client, page=page, details=details, not_free=not_free)
if changed:
touched += 1
fields.update(changed)
process_batch([g for g, _, _ in plan], refresh, "Refreshing")
done = process_batch([g for g, _, _ in plan], refresh, "Refreshing", deadline=deadline)
summary = ", ".join(f"{f} {n}" for f, n in fields.most_common()) or "nothing"
print(f"\n{touched} of {len(plan)} games changed: {summary}")
print(f"\n{touched} of {done} games changed: {summary}")
if not_free:
print(f"::warning::appdetails no longer calls {len(not_free)} game(s) free: "
f"{'; '.join(not_free[:20])}{' …' if len(not_free) > 20 else ''}. The daily purge "
f"removes one only when Steam's store API says so too; if it does not, remove it "
f"in /admin.")
if args.dry_run:
print("Dry run: nothing written.")
return
+172
View File
@@ -0,0 +1,172 @@
"""
A job's time budget and the rotations that keep a daily run the same size
however large the catalogue grows. These pin that a budget is opt-in, that a
loop past its deadline starts no further game, that only the store loops
pause, and that a growing cycle never brings a game's visits closer together.
Run from the repository root: python -m pytest scripts/tests
"""
import zlib
import pytest
from core import budget, fetcher, rotation
from core.budget import Deadline
class Clock:
def __init__(self, now=1000.0):
self.now = now
def __call__(self):
return self.now
@pytest.fixture
def clock(monkeypatch):
c = Clock()
monkeypatch.setattr(budget.time, "monotonic", c)
return c
def _games(n: int) -> list[dict]:
return [{"link": f"https://store.steampowered.com/app/{(i + 1) * 10}/", "name": f"G{i}"} for i in range(n)]
# ──────────── Deadline ────────────
@pytest.mark.parametrize("raw", [None, "", "0", "-5", "soon", "nan", "inf"])
def test_no_budget_means_no_deadline(monkeypatch, raw):
if raw is None:
monkeypatch.delenv(budget.ENV, raising=False)
else:
monkeypatch.setenv(budget.ENV, raw)
assert Deadline.from_env() is None
assert budget.expired(None) is False
def test_a_budget_counts_from_when_it_was_made(monkeypatch, clock):
monkeypatch.setenv(budget.ENV, " 30 ")
deadline = Deadline.from_env()
assert deadline.minutes == 30
clock.now += 29 * 60
assert not budget.expired(deadline)
clock.now += 60
assert budget.expired(deadline) and deadline.used() == pytest.approx(1.0)
def test_the_report_warns_when_a_loop_ran_out_or_came_close(clock):
lines: list[str] = []
deadline = Deadline(10)
clock.now += 60
budget.report("Players", 5, 5, deadline, log=lines.append)
clock.now += 7 * 60
budget.report("Players", 5, 5, deadline, log=lines.append)
budget.report("Players", 3, 5, deadline, log=lines.append)
budget.report("Players", 5, 5, None, log=lines.append)
assert not lines[0].startswith("::warning::") and "1 of 10" in lines[0]
assert lines[1].startswith("::warning::") and "outgrowing" in lines[1]
assert lines[2].startswith("::warning::") and "stopped after 3 of 5" in lines[2]
assert len(lines) == 3
# ──────────── process_batch ────────────
def test_past_the_deadline_no_further_game_is_started(monkeypatch, clock):
monkeypatch.setattr(fetcher.time, "sleep", lambda s: None)
deadline = Deadline(1)
seen: list[str] = []
def fn(game):
seen.append(game["name"])
clock.now += 25 # the third game crosses the minute
assert fetcher.process_batch(_games(10), fn, "Test", deadline=deadline) == 3
assert seen == ["G0", "G1", "G2"]
def test_only_the_store_loops_pause_between_batches(monkeypatch):
monkeypatch.setattr(fetcher, "BATCH_SIZE", 2)
pauses: list[float] = []
monkeypatch.setattr(fetcher.time, "sleep", pauses.append)
assert fetcher.process_batch(_games(5), lambda g: None, "Store") == 5
assert len(pauses) == 2 # between the three batches, not after the last
pauses.clear()
assert fetcher.process_batch(_games(5), lambda g: None, "Players", pause=False) == 5
assert pauses == []
def test_players_are_asked_without_pauses_in_a_daily_order(monkeypatch):
monkeypatch.setattr(fetcher, "STEAM_API_KEY", "key")
monkeypatch.setattr(fetcher.time, "sleep", lambda s: pytest.fail("the Web API loop slept"))
asked: list[str] = []
class Client:
def fetch_player_count(self, appid, key):
asked.append(appid)
return 7
monkeypatch.setattr(fetcher, "get_client", lambda: Client())
games = [dict(g, type_game="online") for g in _games(120)]
fetcher.update_players_only(games, day=100)
first = list(asked)
asked.clear()
fetcher.update_players_only(games, day=101)
assert sorted(first) == sorted(asked) and first != asked
assert all(g["current_players"] == "7" for g in games)
def test_the_full_update_stops_at_its_deadline(monkeypatch, clock):
monkeypatch.setattr(fetcher.time, "sleep", lambda s: None)
monkeypatch.setattr(fetcher, "get_client", lambda: object())
monkeypatch.setattr(fetcher.OverrideGuard, "load", classmethod(lambda cls: None))
fetched: list[str] = []
def fake_fetch(game, client=None, guard=None):
fetched.append(game["name"])
clock.now += 61
monkeypatch.setattr(fetcher, "fetch_full", fake_fetch)
fetcher.update_all_full(_games(4), deadline=Deadline(2))
assert len(fetched) == 2
# ──────────── Rotation ────────────
@pytest.mark.parametrize("base, daily", [(7, 2000), (30, 500), (30, 400)])
def test_a_cycle_is_a_multiple_of_its_base_that_never_shrinks(base, daily):
last = base
for count in range(0, 100_001, 997):
cycle = rotation.cycle_for(count, base, daily)
assert cycle % base == 0 and cycle >= last
assert count / cycle <= daily or cycle == base and count <= base * daily
last = cycle
def test_the_cycle_stays_at_its_base_until_a_day_would_overflow():
assert rotation.cycle_for(9847, 7, 2000) == 7
assert rotation.cycle_for(14000, 7, 2000) == 7
assert rotation.cycle_for(14001, 7, 2000) == 14
assert rotation.cycle_for(12000, 30, 400) == 30
assert rotation.cycle_for(12001, 30, 400) == 60
assert rotation.cycle_for(5, 7, 0) == 7
def test_a_growing_cycle_keeps_every_visit_at_least_a_base_apart():
base = 30
for appid in ("730", "570", "2507950", "4219930"):
crc = zlib.crc32(appid.encode())
visits = []
for day in range(20000, 20400):
cycle = 30 if day < 20150 else 60 if day < 20300 else 90
if rotation.due({"link": f"https://store.steampowered.com/app/{appid}/"}, cycle, day):
visits.append(day)
assert all(day % base == crc % base for day in visits)
assert all(b - a >= base for a, b in zip(visits, visits[1:]))
def test_the_daily_order_holds_within_a_day_and_changes_the_next():
games = _games(50)
assert rotation.daily_order(games, 7) == rotation.daily_order(list(reversed(games)), 7)
assert rotation.daily_order(games, 7) != rotation.daily_order(games, 8)
assert sorted(g["name"] for g in rotation.daily_order(games, 8)) == sorted(g["name"] for g in games)
+58
View File
@@ -0,0 +1,58 @@
"""
check_game_health() is the confirmation every removal goes through: purge
asks it about the games Steam's store API flags, and ingest about every queued
link. These pin how appdetails' answers map to a verdict.
Steam is never called: the client is a fake. Run from the repository root:
python -m pytest scripts/tests
"""
import pytest
from core import health_checker as hc
LINK = "https://store.steampowered.com/app/730/"
class FakeClient:
def __init__(self, status="ok", data=None, head=200):
self.status, self.data, self.head = status, data, head
def fetch_app_details_full(self, appid):
return (self.status, self.data)
def check_store_page(self, appid):
return self.head
@pytest.mark.parametrize("data, expected", [
({"is_free": True}, True),
({"is_free": False, "price_overview": {"initial": 0}}, True),
({"is_free": False, "price_overview": {"initial": 1499}}, False),
({"is_free": False, "price_overview": None}, False),
({}, False),
])
def test_what_appdetails_calls_free(data, expected):
assert hc.free_per_appdetails(data) is expected
@pytest.mark.parametrize("client, status", [
(FakeClient(data={"name": "CS2", "is_free": True}), hc.OK),
(FakeClient(data={"name": "CS2", "is_free": False, "price_overview": {"initial": 999}}), hc.NOT_FREE),
(FakeClient(data={"name": "CS2", "is_free": True, "release_date": {"coming_soon": True}}), hc.COMING_SOON),
(FakeClient(status="unavailable"), hc.UNAVAILABLE),
(FakeClient(status="not_found"), hc.NOT_FOUND_404),
(FakeClient(status="network_error"), hc.NETWORK_ERROR),
(FakeClient(data={}, head=404), hc.NOT_FOUND_404),
(FakeClient(data={}, head=410), hc.NOT_FOUND_410),
(FakeClient(data={}, head=200), hc.OK),
(FakeClient(data={}, head=-1), hc.NETWORK_ERROR),
])
def test_appdetails_answers_map_to_a_verdict(client, status):
result = hc.check_game_health(LINK, "Counter-Strike 2", client)
assert result.status == status and result.appid == "730"
assert result.should_remove is (status in hc.REMOVABLE)
def test_a_link_without_an_appid_is_invalid_without_a_request():
result = hc.check_game_health("https://example.com/", "x", object())
assert result.status == hc.INVALID_FORMAT and result.should_remove
+76
View File
@@ -0,0 +1,76 @@
"""
The leaderboards sample every online (top_online.py) or offline
(top_offline.py) game's player count through the keyed Web API, and
update_data.py refills incomplete records. These run each script end to end
against a fake client: they write their page and save through save_main(),
and the Web API loop never pauses.
Run from the repository root: python -m pytest scripts/tests
"""
import os
import pytest
import top_offline
import top_online
import update_data
from core import budget, data_store, fetcher
def _rec(appid: str, type_game: str, players: str = "10") -> dict:
return {"link": f"https://store.steampowered.com/app/{appid}/", "name": f"Game {appid}",
"type_game": type_game, "status": "active", "current_players": players,
"peak_today": players, "reviews": "90% (Very Positive)", "genre": "Action",
"anti_cheat": "-", "notes": "Not reviewed yet", "last_updated": "2026-01-01T00:00:00Z"}
class Players:
def __init__(self, counts):
self.counts = counts
def fetch_player_count(self, appid, key):
return self.counts.get(appid)
@pytest.fixture
def repo(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.makedirs("data")
monkeypatch.delenv(budget.ENV, raising=False)
monkeypatch.setattr(fetcher, "STEAM_API_KEY", "key")
monkeypatch.setattr(fetcher.time, "sleep", lambda s: pytest.fail("the Web API loop slept"))
data_store.save_main([_rec("10", "online"), _rec("20", "online"), _rec("30", "offline")],
apply_human_overrides=False)
return monkeypatch
def _after() -> dict:
return {data_store.extract_appid(g["link"]): g for g in data_store.load_main()}
def test_top_online_samples_the_online_games_and_writes_its_page(repo):
repo.setattr(top_online, "STEAM_API_KEY", "key")
repo.setattr(fetcher, "get_client", lambda: Players({"10": 50000, "20": 3}))
top_online.main()
page = open(os.path.join("games", "top-online.md"), encoding="utf-8").read()
assert page.index("Game 10") < page.index("Game 20") and "Game 30" not in page
after = _after()
assert (after["10"]["current_players"], after["30"]["current_players"]) == ("50,000", "10")
def test_top_offline_samples_the_offline_games_and_writes_its_page(repo):
repo.setattr(top_offline, "STEAM_API_KEY", "key")
repo.setattr(fetcher, "get_client", lambda: Players({"30": 1200}))
top_offline.main()
page = open(os.path.join("games", "top-offline.md"), encoding="utf-8").read()
assert "Game 30" in page and "Game 10" not in page
assert _after()["30"]["current_players"] == "1,200"
def test_update_data_refills_only_incomplete_records(repo):
refilled: list[str] = []
repo.setattr(fetcher, "get_client", lambda: object())
repo.setattr(fetcher.OverrideGuard, "load", classmethod(lambda cls: None))
repo.setattr(fetcher, "fetch_full", lambda g, client=None, guard=None: refilled.append(g["link"]))
update_data.main()
assert len(refilled) == 3 # the fixture's records lack most store fields
+214
View File
@@ -0,0 +1,214 @@
"""
purge_unhealthy.py removes a game only when two of Steam's answers agree: the
store API's screen of the whole catalogue, and appdetails for each game it
flags. These pin the rules that make a daily, unattended removal safe: what
nobody answered is never "gone", appdetails' unavailable is asked twice,
blocklisted games are not asked about, a broken answer stops the run before
anything is written, and removals are logged as the Worker's reconcile reads
them.
Steam is never called: the client and appdetails are fakes. Run from the
repository root: python -m pytest scripts/tests
"""
import json
import os
import pytest
import purge_unhealthy as pu
from core import budget, data_store
from core.health_checker import (
NETWORK_ERROR, NOT_FREE, OK, UNAVAILABLE, HealthResult,
)
OLD = "2026-01-01T00:00:00Z"
FREE = {"visible": True, "is_free": True, "release": {"steam_release_date": 1}}
def _rec(appid: str, name: str = "") -> dict:
return {"link": f"https://store.steampowered.com/app/{appid}/", "name": name or f"Game {appid}",
"type_game": "online", "notes": "Not reviewed yet", "last_updated": OLD}
def _item(appid: str, **fields) -> dict:
return dict(FREE, id=int(appid), success=1, **fields)
class FakeClient:
"""GetItems items by appid (absent: unknown to Steam) and failed appids."""
def __init__(self, items=None, failed=()):
self.items, self.failed = items or {}, set(failed)
self.asked: list[list[str]] = []
def fetch_items(self, appids, data_request, give_up_after=10):
assert data_request == {"include_release": True}
self.asked.append(list(appids))
return ({a: self.items[a] for a in appids if a in self.items and a not in self.failed},
{a for a in appids if a in self.failed})
@pytest.fixture
def repo(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.makedirs("data")
os.makedirs("scripts")
monkeypatch.delenv(budget.ENV, raising=False)
sleeps: list[float] = []
monkeypatch.setattr(pu.time, "sleep", sleeps.append)
answers: dict[str, list[str]] = {}
health_calls: list[str] = []
def fake_health(link, name="", client=None):
appid = data_store.extract_appid(link)
health_calls.append(appid)
status = answers[appid].pop(0)
return HealthResult(status, appid, name, link)
monkeypatch.setattr(pu, "check_game_health", fake_health)
monkeypatch.setattr(pu, "blocked_appids", lambda: set())
class Repo:
def run(self, records=None, client=None, health=None, argv=()):
if records is not None:
data_store.save_main(records, apply_human_overrides=False)
answers.clear()
answers.update({a: list(v) for a, v in (health or {}).items()})
monkeypatch.setattr(pu, "get_client", lambda: client)
code = pu.main(list(argv))
return code, {data_store.extract_appid(g["link"]) for g in data_store.load_main()}
r = Repo()
r.sleeps, r.health_calls = sleeps, health_calls
r.block = lambda ids: monkeypatch.setattr(pu, "blocked_appids", lambda: set(ids))
return r
def _removed() -> list[dict]:
path = os.path.join("scripts", "removed_games.jsonl")
return [json.loads(line) for line in open(path, encoding="utf-8")] if os.path.exists(path) else []
def _bytes() -> dict:
files = [os.path.join("data", n) for n in sorted(os.listdir("data")) if n.endswith((".jsonl", ".json"))]
files += [os.path.join("scripts", n) for n in sorted(os.listdir("scripts"))]
return {f: open(f, "rb").read() for f in files}
def test_an_empty_catalogue_asks_nothing(repo):
client = FakeClient()
assert repo.run(None, client) == (0, set())
assert client.asked == []
def test_a_healthy_catalogue_asks_appdetails_nothing(repo):
client = FakeClient(items={"730": _item("730"), "570": _item("570")})
code, live = repo.run([_rec("730"), _rec("570")], client)
assert (code, live) == (0, {"730", "570"})
assert repo.health_calls == [] and _removed() == []
def test_a_game_both_answers_flag_is_removed_and_logged(repo):
client = FakeClient(items={"730": _item("730"), "4991340": _item("4991340", unlisted=True, is_free=False)})
code, live = repo.run([_rec("730"), _rec("4991340", "Might & Poison Mushroom")], client,
health={"4991340": [NOT_FREE]})
assert (code, live) == (0, {"730"})
[row] = _removed()
assert (row["appid"], row["status_code"], row["reason"]) == ("4991340", "not_free", "No longer free-to-play")
assert row["name"] == "Might & Poison Mushroom" and row["removed_at"]
def test_a_flag_appdetails_contradicts_is_kept(repo):
client = FakeClient(items={"2878670": _item("2878670", is_free=False)})
code, live = repo.run([_rec("2878670", "Spoop Troop")], client, health={"2878670": [OK]})
assert (code, live) == (0, {"2878670"}) and _removed() == []
def test_unavailable_is_asked_again_before_a_game_goes(repo):
client = FakeClient(items={}) # answered, and unknown to Steam
code, live = repo.run([_rec("10"), _rec("20")], client,
health={"10": [UNAVAILABLE, UNAVAILABLE], "20": [UNAVAILABLE, OK]})
assert code == 0 and live == {"20"}
assert repo.health_calls == ["10", "20", "10", "20"]
assert len(repo.sleeps) == 1 and 0 < repo.sleeps[0] <= pu.RECHECK_AFTER
assert [r["status_code"] for r in _removed()] == ["unavailable"]
def test_what_nobody_answered_is_never_gone(repo):
client = FakeClient(items={"730": _item("730")}, failed={"570"})
code, live = repo.run([_rec("730"), _rec("570"), _rec("440")], client,
health={"440": [NETWORK_ERROR]})
assert (code, live) == (0, {"730", "570", "440"})
assert repo.health_calls == ["440"] # 570's batch failed: not even asked
def test_no_answer_at_all_writes_nothing(repo):
repo.run([_rec("730")], FakeClient(items={"730": _item("730")}))
before = _bytes()
code, live = repo.run(None, FakeClient(failed={"730"}))
assert (code, live) == (0, {"730"}) and _bytes() == before
def test_a_blocklisted_game_is_not_asked_about(repo):
repo.block({"570"})
client = FakeClient(items={"730": _item("730")})
repo.run([_rec("730"), _rec("570")], client)
assert "570" not in client.asked[0] and repo.health_calls == []
def test_too_many_flags_stop_the_run_before_any_request(repo):
repo.run([_rec("730"), _rec("570"), _rec("440")], FakeClient(items={a: _item(a) for a in ("730", "570", "440")}))
before = _bytes()
code, _ = repo.run(None, FakeClient(items={}), argv=["--max-candidates", "2"])
assert code == 2 and repo.health_calls == [] and _bytes() == before
def test_too_many_removals_stop_the_run_before_anything_is_written(repo):
repo.run([_rec("730"), _rec("570")], FakeClient(items={"730": _item("730"), "570": _item("570")}))
before = _bytes()
client = FakeClient(items={"730": _item("730", is_free=False), "570": _item("570", is_free=False)})
code, live = repo.run(None, client, health={"730": [NOT_FREE], "570": [NOT_FREE]},
argv=["--max-removals", "1"])
assert code == 2 and live == {"730", "570"} and _bytes() == before
def test_a_dry_run_writes_nothing(repo):
repo.run([_rec("730")], FakeClient(items={"730": _item("730")}))
before = _bytes()
code, _ = repo.run(None, FakeClient(items={"730": _item("730", unlisted=True)}),
health={"730": [NOT_FREE]}, argv=["--dry-run"])
assert code == 0 and _bytes() == before
def test_a_removal_steam_has_undone_is_named(repo, capsys):
with open(os.path.join("scripts", "removed_games.jsonl"), "w", encoding="utf-8") as f:
for appid, code in (("4214820", "unavailable"), ("620", "not_free"), ("99", "admin_removed")):
f.write(json.dumps({"appid": appid, "name": f"Old {appid}", "status_code": code,
"removed_at": OLD}) + "\n")
client = FakeClient(items={"730": _item("730"), "4214820": _item("4214820"),
"620": _item("620", is_free=False), "99": _item("99")})
repo.run([_rec("730")], client)
out = capsys.readouterr().out
assert "::notice::1 removed game(s)" in out and "4214820 Old 4214820" in out
assert "620" not in out.split("::notice::")[1] and "Old 99" not in out
def test_the_budget_leaves_the_rest_unasked(repo, monkeypatch):
client = FakeClient(items={"730": _item("730", unlisted=True)})
monkeypatch.setenv(budget.ENV, "0.000001")
code, live = repo.run([_rec("730")], client, health={"730": [NOT_FREE]})
assert (code, live) == (0, {"730"}) and repo.health_calls == []
@pytest.mark.parametrize("item, why", [
(None, "unknown"),
({"visible": False, "is_free": True}, "hidden"),
({"visible": True, "unlisted": True}, "unlisted"),
({"visible": True, "is_free": True, "release": {"is_coming_soon": True}}, "coming_soon"),
({"visible": True}, "not_free"),
({"is_free": True, "type": 2}, None), # a free mod: type is not read
(FREE, None),
])
def test_what_the_screen_flags(item, why):
from core.store_items import screen_item
assert screen_item(item) == why
+98 -1
View File
@@ -1,8 +1,15 @@
"""
refetch_all.py clears what Steam can give back and keeps what a person
decided. Run from the repository root: python -m pytest scripts/tests
decided, one game at a time: a game appdetails does not answer for, or the
ones a part cut short by its budget never reached, stay exactly as they were.
Run from the repository root: python -m pytest scripts/tests
"""
import os
import pytest
import refetch_all
from core import budget, data_store
def test_clear_fetchable_resets_a_default_to_the_skeleton_s_value():
@@ -19,3 +26,93 @@ def test_clear_fetchable_keeps_a_decision():
refetch_all.clear_fetchable(game)
assert (game["anti_cheat"], game["anti_cheat_note"], game["is_kernel_ac"], game["genre"]) == \
("BattlEye", "BattlEye [Kernel]", True, "FPS")
# ──────────── In parts, game by game ────────────
def _rec(appid: str, **fields) -> dict:
base = {"link": f"https://store.steampowered.com/app/{appid}/", "name": f"Game {appid}",
"description": "kept", "tags": ["FPS"], "reviews": "50% (Mixed)", "genre": "FPS",
"anti_cheat": "-", "anti_cheat_note": "", "is_kernel_ac": None, "type_game": "online",
"notes": "Not reviewed yet", "last_updated": "2026-01-01T00:00:00Z"}
base.update(fields)
return base
class _Client:
def __init__(self, failing=()):
self.failing = set(failing)
self.asked: list[str] = []
def fetch_app_details_full(self, appid):
self.asked.append(appid)
if appid in self.failing:
return ("network_error", None)
return ("ok", {"name": f"Fresh {appid}", "is_free": True})
@pytest.mark.parametrize("text, part", [("1/1", (1, 1)), (" 3 / 8 ", (3, 8)), ("8/8", (8, 8))])
def test_a_part_is_read(text, part):
assert refetch_all.parse_part(text) == part
@pytest.mark.parametrize("text", ["0/4", "5/4", "1/0", "x", "", "1/2/3"])
def test_a_bad_part_is_refused(text):
with pytest.raises(Exception):
refetch_all.parse_part(text)
def test_the_parts_split_the_catalogue():
games = [_rec(str(10 * i)) for i in range(1, 501)]
counts = [sum(refetch_all.in_part(g, (k, 8)) for g in games) for k in range(1, 9)]
assert sum(counts) == len(games) and min(counts) > 0
def test_a_game_appdetails_does_not_answer_for_is_left_as_it_was(monkeypatch):
calls: list[str] = []
monkeypatch.setattr(refetch_all, "fetch_full",
lambda g, client=None, prefetched_details=None, guard=None: calls.append(g["link"]))
game = _rec("730")
before = dict(game)
assert refetch_all.refetch_one(game, _Client(failing={"730"}), None) is False
assert game == before and calls == []
def test_a_game_is_cleared_and_fetched_on_a_copy(monkeypatch):
def fake_fetch(g, client=None, prefetched_details=None, guard=None):
assert g["description"] == "" and g["reviews"] == "N/A" # cleared before the fetch
g["name"] = prefetched_details["name"]
monkeypatch.setattr(refetch_all, "fetch_full", fake_fetch)
game = _rec("730", anti_cheat="BattlEye")
assert refetch_all.refetch_one(game, _Client(), None) is True
assert game["name"] == "Fresh 730" and game["anti_cheat"] == "BattlEye"
def test_a_cut_part_leaves_every_other_game_alone_and_resumes_oldest_first(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.makedirs("data")
games = [_rec(str(10 * i), last_updated=f"2026-01-{i:02d}T00:00:00Z") for i in range(1, 9)]
data_store.save_main(games, apply_human_overrides=False)
client = _Client()
monkeypatch.setattr(refetch_all, "get_client", lambda: client)
monkeypatch.setattr(refetch_all.OverrideGuard, "load", classmethod(lambda cls: None))
monkeypatch.setattr(refetch_all, "STEAM_API_KEY", "")
clock = [0.0]
monkeypatch.setattr(budget.time, "monotonic", lambda: clock[0])
def fake_fetch(g, client=None, prefetched_details=None, guard=None):
clock[0] += 40
g["name"] = prefetched_details["name"]
g["last_updated"] = "2026-10-23T00:00:00Z"
monkeypatch.setattr(refetch_all, "fetch_full", fake_fetch)
monkeypatch.setattr("core.fetcher.time.sleep", lambda s: None)
monkeypatch.setenv(budget.ENV, "1")
refetch_all.main(["--part", "1/1"])
after = {data_store.extract_appid(g["link"]): g for g in data_store.load_main()}
assert client.asked == ["10", "20"] # the two oldest, then the budget ran out
assert after["10"]["name"] == "Fresh 10" and after["30"]["name"] == "Game 30"
assert after["30"]["description"] == "kept" # never cleared
+47
View File
@@ -239,3 +239,50 @@ def test_refetch_all_never_clears_it():
# The other player fields are cleared and fetched again; this one cannot be.
assert "all_time_peak" not in refetch_all.CLEARABLE
assert "all_time_peak" not in refetch_all.CONDITIONAL
def test_the_budget_stops_the_steamcharts_part_and_keeps_what_was_raised(monkeypatch):
from core import budget
clock = [0.0]
monkeypatch.setattr(budget.time, "monotonic", lambda: clock[0])
deadline = budget.Deadline(1)
games = [_game(str(i)) for i in range(1, 6)]
class Slow(_Client):
def fetch(self, appid):
clock[0] += 25
return super().fetch(appid)
client = Slow({str(i): Result("ok", 100 * i) for i in range(1, 6)})
lines: list[str] = []
tally = rp.raise_from_steamcharts(games, client, log=lines.append, deadline=deadline)
assert client.asked == ["1", "2", "3"]
assert tally["raised"] == 3 and tally["skipped"] == 2
assert any(line.startswith("::warning::") for line in lines)
def test_a_larger_catalogue_is_asked_less_often_never_more():
assert rp.cycle_for(9847, rp.PEAK_CYCLE_DAYS, rp.PEAK_DAILY) == 30
assert rp.cycle_for(20000, rp.PEAK_CYCLE_DAYS, rp.PEAK_DAILY) == 60
def test_a_run_asks_today_s_slice_and_saves(tmp_path, monkeypatch, capsys):
import os
from core import budget, data_store, rotation
monkeypatch.chdir(tmp_path)
os.makedirs("data")
monkeypatch.delenv(budget.ENV, raising=False)
games = [_game(str(10 * i), current_players="0", peak_today="0") for i in range(1, 41)]
data_store.save_main(games, apply_human_overrides=False)
day = rotation.slice_of("10", rp.PEAK_CYCLE_DAYS) + 30 * 700
client = _Client({str(10 * i): Result("ok", 1000 + i) for i in range(1, 41)})
monkeypatch.setattr(rp, "SteamChartsClient", lambda: client)
rp.main(["--day", str(day)])
assert "10" in client.asked
assert all(rotation.slice_of(a, 30) == day % 30 for a in client.asked)
after = {data_store.extract_appid(g["link"]): g for g in data_store.load_main()}
assert after["10"]["all_time_peak"] == "1,001"
assert f"slice {day % 30}/30" in capsys.readouterr().out
+59
View File
@@ -391,3 +391,62 @@ def test_a_blurb_that_now_fits_further_is_written_but_is_not_news():
def test_a_rewritten_blurb_is_a_change():
game = _game(description="Old words, cut short…")
assert "description" in rsd.apply_app_details(game, {"short_description": "New words entirely."})
# ──────────── A growing catalogue ────────────
def test_the_page_cycle_grows_with_the_catalogue_and_keeps_its_weekday():
assert rsd.cycle_for(9847, rsd.PAGE_CYCLE_DAYS, rsd.PAGE_DAILY) == 7
games = [_game(str(10 * i)) for i in range(1, 12001)]
assert rsd.cycle_for(len(games), rsd.PAGE_CYCLE_DAYS, 1000) == 14
for day in range(20000, 20014):
plan = rsd.build_plan(games, day, want_details=False, page_cycle=14)
assert len(plan) < 1000
assert all(slice_of(rsd.extract_appid(g["link"]), 7) == day % 7 for g, _, _ in plan)
def test_the_slice_comes_in_an_order_that_changes_daily():
games = [_game(str(10 * i)) for i in range(1, 701)]
first = [g["link"] for g, _, _ in rsd.build_plan(games, 20006, want_details=False)]
again = [g["link"] for g, _, _ in rsd.build_plan(list(reversed(games)), 20006, want_details=False)]
week_later = [g["link"] for g, _, _ in rsd.build_plan(games, 20013, want_details=False)]
assert first == again
assert sorted(first) == sorted(week_later) and first != week_later
def test_a_game_appdetails_no_longer_call_free_is_listed_not_written():
game = _game()
before = dict(game)
listed: list[str] = []
details = {"name": "Counter-Strike 2", "is_free": False,
"price_overview": {"initial": 1499, "final": 1499}}
rsd.refresh_game(game, FakeClient(details=details), page=False, details=True, not_free=listed)
assert listed == ["730 Counter-Strike 2"]
assert game["notes"] == before["notes"]
listed.clear()
rsd.refresh_game(_game(), FakeClient(details={"name": "x", "is_free": True}), page=False, details=True,
not_free=listed)
assert listed == []
def test_a_run_refreshes_today_s_slice_and_lists_what_appdetails_calls_paid(tmp_path, monkeypatch, capsys):
import os
from core import budget, data_store
monkeypatch.chdir(tmp_path)
os.makedirs("data")
monkeypatch.delenv(budget.ENV, raising=False)
monkeypatch.setattr("core.fetcher.time.sleep", lambda s: None)
games = [_game(str(10 * i), name=f"Game {i}") for i in range(1, 61)]
data_store.save_main(games, apply_human_overrides=False)
client = FakeClient(details={"name": "Renamed", "is_free": False, "price_overview": {"initial": 999}})
monkeypatch.setattr(rsd, "get_client", lambda: client)
rsd.main(["--only", "details", "--day", "20030"])
out = capsys.readouterr().out
asked = [c.split(":")[1] for c in client.calls]
assert asked and all(rsd.due(_game(a), 30, 20030) for a in asked)
assert "slice 20/30" in out
assert "::warning::appdetails no longer calls" in out
after = {data_store.extract_appid(g["link"]): g for g in data_store.load_main()}
assert all(after[a]["name"] == "Renamed" for a in asked)
+17
View File
@@ -132,6 +132,23 @@ def test_anticheat_items_survive_a_failed_batch(monkeypatch):
assert sent["data_request"] == {"include_extra_details": True}
def test_items_pass_the_data_request_through_and_tell_unknown_from_unanswered(monkeypatch):
monkeypatch.setattr(steam_client, "STORE_ITEMS_BATCH", 2)
answered = {"response": {"store_items": [
{"id": 730, "success": 1, "reviews": {}},
{"id": 1, "appid": 0, "success": 15}, # unknown to Steam
]}}
client, session = _client(_Resp(steam_client.STORE_ITEMS_URL, body=answered),
_Resp(steam_client.STORE_ITEMS_URL, status_code=500))
monkeypatch.setattr(steam_client, "MAX_RETRIES", 1)
items, failed = client.fetch_items(["730", "1", "570"], {"include_reviews": True})
assert list(items) == ["730"]
assert failed == {"570"} # "1" was answered for: Steam does not know it
sent = json.loads(session.requests[0]["params"]["input_json"])
assert sent["data_request"] == {"include_reviews": True}
assert sent["context"]["country_code"] == "US"
def test_anticheat_items_give_up_after_failures_in_a_row(monkeypatch):
monkeypatch.setattr(steam_client, "STORE_ITEMS_BATCH", 1)
bad = [_Resp(steam_client.STORE_ITEMS_URL, status_code=403) for _ in range(2)]
+218
View File
@@ -0,0 +1,218 @@
"""
update_reviews.py reads every game's review summary from GetItems in a
hundred-odd requests. These pin what makes that safe to run daily: Steam's
floored percentage is written in the stored shape, a count of 0 is never
trusted on its own, an unanswered game keeps its value, last_updated moves
only on a change or on the heartbeat that keeps the site's "stale" badge
quiet, and a second run writes the same bytes.
Steam is never called: the client is a fake. Run from the repository root:
python -m pytest scripts/tests
"""
import os
import zlib
from datetime import datetime, timedelta, timezone
import pytest
import update_reviews as ur
from core import budget, data_store
from core.fetcher import reviews_from_summary
STAMP = "2026-10-23T06:30:00Z"
OLD = "2026-01-01T00:00:00Z"
NOW = datetime(2026, 10, 23, 6, 30, tzinfo=timezone.utc)
def _recent(days: int = 1) -> str:
return (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")
def _rec(appid: str, reviews: str, last_updated: str = OLD) -> dict:
return {"link": f"https://store.steampowered.com/app/{appid}/", "name": f"Game {appid}",
"reviews": reviews, "type_game": "online", "notes": "Not reviewed yet",
"last_updated": last_updated}
def _item(appid: str, count=None, pct=None, label=None) -> dict:
item = {"id": int(appid), "success": 1}
if count is not None:
summary = {"review_count": count, "review_score_label": label}
if pct is not None:
summary["percent_positive"] = pct
item["reviews"] = {"summary_filtered": summary}
return item
class FakeClient:
"""GetItems items by appid (absent: Steam does not know it), the appids
whose batch failed, and appreviews' query_summary by appid (None: failed)."""
def __init__(self, items=None, failed=(), appreviews=None):
self.items, self.failed = items or {}, set(failed)
self.appreviews = appreviews or {}
self.asked: list[str] = []
def fetch_items(self, appids, data_request, give_up_after=10):
assert data_request == {"include_reviews": True}
return ({a: self.items[a] for a in appids if a in self.items and a not in self.failed},
{a for a in appids if a in self.failed})
def fetch_reviews(self, appid):
self.asked.append(appid)
return self.appreviews.get(appid)
@pytest.fixture
def repo(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
os.makedirs("data")
monkeypatch.setattr(ur, "now_iso", lambda: STAMP)
monkeypatch.delenv(budget.ENV, raising=False)
monkeypatch.setattr("core.fetcher.time.sleep", lambda s: None)
def run(records=None, client=None, argv=()):
if records is not None:
data_store.save_main(records, apply_human_overrides=False)
monkeypatch.setattr(ur, "get_client", lambda: client)
ur.main(list(argv))
return {data_store.extract_appid(g["link"]): g for g in data_store.load_main()}
return run
def _bytes() -> dict:
return {n: open(os.path.join("data", n), "rb").read()
for n in sorted(os.listdir("data")) if n.endswith((".jsonl", ".json"))}
# ──────────── The stored shape ────────────
@pytest.mark.parametrize("pos, total, expected", [
(29, 100, "29% (Mixed)"), # floor(29/100*100) in floating point is 28
(2, 3, "66% (Mixed)"),
(999, 1000, "99% (Mixed)"),
(0, 0, "No reviews"),
])
def test_appreviews_is_floored_in_integers(pos, total, expected):
summary = {"total_positive": pos, "total_reviews": total, "review_score_desc": "Mixed"}
assert reviews_from_summary(summary) == expected
@pytest.mark.parametrize("stored, item, expected", [
("86% (Very Positive)", _item("1", 9943066, 85, "Very Positive"), (ur.SET, "85% (Very Positive)")),
("N/A", _item("1", 3, 100, "3 user reviews"), (ur.SET, "100% (3 user reviews)")),
("N/A", _item("1", 12, 50, ""), (ur.SET, "50%")),
("No reviews", _item("1", 0, 0, "No user reviews"), (ur.KEEP, None)),
("93% (Very Positive)", _item("1", 0, 0, "No user reviews"), (ur.ASK, None)),
("N/A", _item("1", 0, 0, "No user reviews"), (ur.ASK, None)),
("", _item("1", 0), (ur.ASK, None)),
("50% (Mixed)", None, (ur.UNKNOWN, None)),
("50% (Mixed)", _item("1"), (ur.UNKNOWN, None)), # no summary
("50% (Mixed)", _item("1", 40, None, "Mixed"), (ur.UNKNOWN, None)), # count without a share
("50% (Mixed)", _item("1", 40, 101, "Mixed"), (ur.UNKNOWN, None)),
("50% (Mixed)", {"reviews": {"summary_filtered": {"percent_positive": 50}}}, (ur.UNKNOWN, None)),
])
def test_what_an_item_says_to_do(stored, item, expected):
assert ur.decide(stored, item) == expected
def test_the_heartbeat_spreads_over_a_week_and_reads_a_bad_stamp_as_due():
game = _rec("730", "85% (Very Positive)")
days = ur.HEARTBEAT_DAYS + zlib.crc32(b"730") % ur.HEARTBEAT_JITTER_DAYS
assert 14 <= days <= 20
game["last_updated"] = (NOW - timedelta(days=days - 1)).strftime("%Y-%m-%dT%H:%M:%SZ")
assert not ur.heartbeat_due(game, NOW)
game["last_updated"] = (NOW - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")
assert ur.heartbeat_due(game, NOW)
game["last_updated"] = ""
assert ur.heartbeat_due(game, NOW)
# ──────────── A run ────────────
def test_a_changed_summary_is_written_and_stamped(repo):
client = FakeClient(items={"730": _item("730", 100, 85, "Very Positive")})
out = repo([_rec("730", "86% (Very Positive)", _recent())], client)
assert out["730"]["reviews"] == "85% (Very Positive)"
assert out["730"]["last_updated"] == STAMP
assert client.asked == []
def test_an_unchanged_game_keeps_its_stamp_until_its_heartbeat(repo):
recent, old = _recent(), _recent(days=40)
client = FakeClient(items={"730": _item("730", 100, 85, "Very Positive"),
"570": _item("570", 100, 80, "Very Positive")})
out = repo([_rec("730", "85% (Very Positive)", recent), _rec("570", "80% (Very Positive)", old)], client)
assert out["730"]["last_updated"] == recent
assert out["570"]["last_updated"] == STAMP # read, and due its heartbeat
def test_a_second_run_writes_the_same_bytes(repo):
client = FakeClient(items={"730": _item("730", 100, 85, "Very Positive"),
"570": _item("570", 0, 0, "No user reviews")},
appreviews={"570": {"total_reviews": 4, "total_positive": 3,
"review_score_desc": "4 user reviews"}})
repo([_rec("730", "86% (Very Positive)"), _rec("570", "N/A")], client)
first = _bytes()
out = repo(None, client)
assert _bytes() == first
assert out["570"]["reviews"] == "75% (4 user reviews)"
def test_a_zero_is_checked_on_appreviews_before_a_percentage_goes(repo):
zero = _item("845070", 0, 0, "No user reviews")
failing = FakeClient(items={"845070": zero}, appreviews={"845070": None})
out = repo([_rec("845070", "93% (Very Positive)")], failing)
assert out["845070"]["reviews"] == "93% (Very Positive)"
assert out["845070"]["last_updated"] == OLD # not read: no heartbeat either
assert failing.asked == ["845070"]
answered = FakeClient(items={"845070": zero}, appreviews={"845070": {
"total_reviews": 5843, "total_positive": 5318, "review_score_desc": "Very Positive"}})
assert repo(None, answered)["845070"]["reviews"] == "91% (Very Positive)"
gone = FakeClient(items={"845070": zero}, appreviews={"845070": {"total_reviews": 0}})
assert repo(None, gone)["845070"]["reviews"] == "No reviews"
def test_no_reviews_stays_without_asking(repo):
client = FakeClient(items={"4263530": _item("4263530", 0, 0, "No user reviews")})
out = repo([_rec("4263530", "No reviews", _recent())], client)
assert out["4263530"]["reviews"] == "No reviews" and client.asked == []
def test_an_unanswered_game_keeps_everything(repo):
client = FakeClient(items={"730": _item("730", 100, 85, "Very Positive"),
"570": _item("570", 100, 70, "Mostly Positive")}, failed={"570"})
out = repo([_rec("730", "86% (Very Positive)"), _rec("570", "80% (Very Positive)")], client)
assert out["570"]["reviews"] == "80% (Very Positive)" and out["570"]["last_updated"] == OLD
assert out["730"]["reviews"] == "85% (Very Positive)"
def test_no_answer_at_all_writes_nothing(repo):
repo([_rec("730", "86% (Very Positive)"), _rec("570", "80% (Very Positive)")], FakeClient())
before = _bytes()
repo(None, FakeClient(failed={"730", "570"}))
assert _bytes() == before
def test_appreviews_is_capped_and_stops_at_the_deadline(repo, monkeypatch):
zero = {a: _item(a, 0, 0, "No user reviews") for a in ("10", "20", "30")}
summary = {"total_reviews": 1, "total_positive": 1, "review_score_desc": "1 user reviews"}
client = FakeClient(items=zero, appreviews={a: summary for a in zero})
monkeypatch.setattr(ur, "ASK_CAP", 2)
out = repo([_rec(a, "N/A") for a in zero], client)
assert len(client.asked) == 2
assert sum(g["reviews"] == "100% (1 user reviews)" for g in out.values()) == 2
client = FakeClient(items=zero, appreviews={a: summary for a in zero})
monkeypatch.setenv(budget.ENV, "0.000001")
repo(None, client)
assert client.asked == []
def test_a_dry_run_writes_nothing(repo):
repo([_rec("730", "86% (Very Positive)")], FakeClient())
before = _bytes()
repo(None, FakeClient(items={"730": _item("730", 100, 85, "Very Positive")}), argv=["--dry-run"])
assert _bytes() == before
+3 -1
View File
@@ -12,6 +12,7 @@ Output: games/top-offline.md (10 columns, same shape as top-online.md).
import sys, os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from datetime import datetime, timezone
from core.budget import Deadline
from core.constants import STEAM_API_KEY, GAMES_DIR, TOP_OFFLINE_LIMIT
from core.data_store import load_main, save_main, extract_appid
from core.fetcher import update_players_only
@@ -58,6 +59,7 @@ def main():
if not STEAM_API_KEY:
print("No Steam API key (set STEAM_API_KEY).")
sys.exit(1)
deadline = Deadline.from_env()
games = load_main()
offline = [g for g in games if _is_offline(g) and g.get("status", "active") == "active"]
if not offline:
@@ -68,7 +70,7 @@ def main():
for g in offline:
aid = extract_appid(g.get("link", ""))
if aid: prev[aid] = _ppc(g.get("current_players", "0"))
update_players_only(games, type_filter="offline")
update_players_only(games, type_filter="offline", deadline=deadline)
# Re-filter after update (status may have changed).
offline = [g for g in games if _is_offline(g) and g.get("status", "active") == "active"]
offline.sort(key=lambda g: _ppc(g.get("current_players", "0")), reverse=True)
+3 -1
View File
@@ -3,6 +3,7 @@
import sys, os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from datetime import datetime, timezone
from core.budget import Deadline
from core.constants import STEAM_API_KEY, GAMES_DIR, TOP_ONLINE_LIMIT
from core.data_store import load_main, save_main, extract_appid
from core.fetcher import update_players_only
@@ -39,6 +40,7 @@ def _stars(r):
def main():
if not STEAM_API_KEY: print("No key."); sys.exit(1)
deadline = Deadline.from_env()
games = load_main()
online = [g for g in games if g.get("type_game","").lower()=="online"
and g.get("status","active")=="active"]
@@ -48,7 +50,7 @@ def main():
for g in online:
aid = extract_appid(g.get("link",""))
if aid: prev[aid] = _ppc(g.get("current_players","0"))
update_players_only(games)
update_players_only(games, deadline=deadline)
# Re-filter after update (status may have changed)
online = [g for g in games if g.get("type_game","").lower()=="online"
and g.get("status","active")=="active"]
+3 -1
View File
@@ -2,14 +2,16 @@
"""Full data update – enriches incomplete records."""
import sys, os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.budget import Deadline
from core.data_store import load_main, save_main, migrate_record
from core.fetcher import update_all_full
def main():
deadline = Deadline.from_env()
games = load_main()
print(f"Loaded {len(games)} games")
for g in games: migrate_record(g)
update_all_full(games)
update_all_full(games, deadline=deadline)
save_main(games)
print(f"\n✓ Saved {len(games)} games")
+175 -10
View File
@@ -1,15 +1,180 @@
#!/usr/bin/env python3
"""Update reviews only. Every 2 days."""
import sys, os
"""
Refresh every game's review summary from Steam's own, a hundred-odd requests a day.
This used to ask store.steampowered.com/appreviews about one game at a time:
two seconds a game, five and a half hours at 9,800 games, and on 2026-10-23 it
ran into GitHub's six-hour limit and was stopped with nothing saved.
IStoreBrowseService/GetItems answers for fifty games a request with the summary
the store shows (core/store_items.review_summary): the whole catalogue in about
two minutes.
The rules:
- A summary with reviews is written "85% (Very Positive)": Steam's floored
percentage and its label, the vocabulary appreviews has ("3 user
reviews", "Mixed").
- A count of 0 is never written on GetItems' word alone: its summary has
said 0 for a game with thousands of reviews. A game stored as "No
reviews" stays so; one stored with a percentage, "N/A" or nothing is asked
on appreviews instead (at most ASK_CAP a run, in an order that changes
daily), and that answer is written, "No reviews" included.
- No item, no usable summary, a failed batch or a failed appreviews request
leaves the stored value alone. When GetItems answers for no game at all,
nothing is written.
- last_updated moves when the value changes, and on a heartbeat: a game
whose summary was read and whose stamp is 14 to 20 days old (crc32
jitter, so the stamps spread out) is stamped again. The site calls a
record older than 30 days stale (web/src/lib/validation.ts, /health),
and this job used to stamp every game on every run; the heartbeat keeps
that meaning - read by the pipeline lately - without rewriting every
record every day.
- Everything is written by save_main().
Usage (from the repository root):
python scripts/update_reviews.py
python scripts/update_reviews.py --limit 200 --dry-run
"""
import argparse
import os
import sys
import zlib
from collections import Counter
from datetime import datetime, timedelta, timezone
from typing import Optional
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from core.data_store import load_main, save_main
from core.fetcher import update_reviews_only
def main():
from core.budget import Deadline
from core.data_store import extract_appid, load_main, migrate_record, now_iso, save_main
from core.fetcher import process_batch, reviews_from_summary, reviews_text
from core.normalize import normalize_records
from core.rotation import daily_order, day_number
from core.steam_client import get_client
from core.store_items import review_summary
ASK_CAP = 300
HEARTBEAT_DAYS = 14
HEARTBEAT_JITTER_DAYS = 7
NO_REVIEWS = "No reviews"
# What decide() says to do with one game.
SET = "set" # write this value
ASK = "ask" # ask appreviews: GetItems said 0
KEEP = "keep" # read, and nothing to write
UNKNOWN = "unknown" # not read: no item or no usable summary
def decide(stored: str, item: Optional[dict]) -> tuple[str, Optional[str]]:
"""What GetItems' item says to do with a game whose reviews are `stored`."""
found = review_summary(item)
if found is None:
return UNKNOWN, None
count, pct, label = found
if count > 0:
return SET, reviews_text(count, pct, label)
return (KEEP, None) if stored == NO_REVIEWS else (ASK, None)
def heartbeat_due(game: dict, now: datetime) -> bool:
"""Whether a game read today is due a new stamp although nothing changed:
its last_updated is 14 to 20 days old (by its appid), or unreadable."""
appid = extract_appid(game.get("link", "")) or ""
days = HEARTBEAT_DAYS + zlib.crc32(appid.encode()) % HEARTBEAT_JITTER_DAYS
try:
stamp = datetime.strptime(game.get("last_updated") or "", "%Y-%m-%dT%H:%M:%SZ")
except ValueError:
return True
return now - stamp.replace(tzinfo=timezone.utc) >= timedelta(days=days)
def record(game: dict, value: Optional[str], now: datetime, tally: Counter) -> None:
"""A game whose summary was read: write `value` (None: nothing new) and
stamp it as the rules say."""
if value is not None and game.get("reviews") != value:
game["reviews"] = value
game["last_updated"] = now_iso()
tally["changed"] += 1
elif heartbeat_due(game, now):
game["last_updated"] = now_iso()
tally["heartbeat"] += 1
def refresh(games: list[dict], client, deadline: Optional[Deadline] = None,
day: Optional[int] = None, now: Optional[datetime] = None) -> Counter:
"""Refresh the reviews of `games` in place. Returns a tally: changed,
heartbeat, unknown, unanswered (batches GetItems failed), asked,
ask_failed, ask_skipped (over the cap or past the deadline), and
no_answer when GetItems answered for nobody."""
now = now or datetime.now(timezone.utc)
by_id = {}
for game in games:
appid = extract_appid(game.get("link", ""))
if appid:
by_id[appid] = game
items, failed = client.fetch_items(list(by_id), {"include_reviews": True})
tally: Counter = Counter(unanswered=len(failed))
if by_id and len(failed) == len(by_id):
tally["no_answer"] = 1
return tally
asks: list[dict] = []
for appid, game in by_id.items():
if appid in failed:
continue
kind, value = decide(game.get("reviews", ""), items.get(appid))
if kind == UNKNOWN:
tally["unknown"] += 1
elif kind == ASK:
asks.append(game)
else:
record(game, value, now, tally)
def ask(game: dict) -> None:
summary = client.fetch_reviews(extract_appid(game["link"]))
if summary is None:
tally["ask_failed"] += 1
else:
record(game, reviews_from_summary(summary), now, tally)
asks = daily_order(asks, day_number() if day is None else day)
done = process_batch(asks[:ASK_CAP], ask, "Reviews from appreviews", deadline=deadline) if asks else 0
tally["asked"] = done
tally["ask_skipped"] = len(asks) - done
return tally
def main(argv: Optional[list[str]] = None) -> None:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0].strip())
ap.add_argument("--limit", type=int, help="at most this many games (for trying it out)")
ap.add_argument("--dry-run", action="store_true", help="ask Steam and report, write nothing")
args = ap.parse_args(argv)
deadline = Deadline.from_env()
games = load_main()
print(f"{len(games)} games – updating reviews...")
update_reviews_only(games)
save_main(games)
print(f"✓ Done.")
for g in games:
migrate_record(g)
# Compared as stored, like the other refresh jobs.
normalize_records(games)
targets = games if args.limit is None else games[: args.limit]
print(f"{len(games)} games; reading the review summaries of {len(targets)} from Steam")
if __name__ == "__main__": main()
tally = refresh(targets, get_client(), deadline=deadline)
if tally["no_answer"]:
print("::warning::GetItems answered for no game: nothing written.")
return
print(f"\nReviews: {tally['changed']} changed, {tally['heartbeat']} re-stamped (heartbeat), "
f"{tally['unknown']} without a usable summary; appreviews asked about {tally['asked']} "
f"({tally['ask_failed']} failed, {tally['ask_skipped']} left for another day)")
if tally["unanswered"]:
print(f"::warning::GetItems did not answer for {tally['unanswered']} games; "
f"they keep their reviews until the next run.")
if args.dry_run:
print("Dry run: nothing written.")
return
save_main(games)
print(f"✓ Saved {len(games)} games")
if __name__ == "__main__":
main()
+1 -1
View File
@@ -194,7 +194,7 @@
</Button>
<p class="text-xs text-muted-foreground">
A batch waits while the previous one is still in scripts/temp_info.jsonl, or while a data job is running or
waiting its turn - pushing then would cancel that job.
waiting its turn - the ingest it starts would only queue behind that job.
</p>
</article>
+2 -2
View File
@@ -71,8 +71,8 @@
<div>
<h1 class="text-2xl font-semibold tracking-tight">Pipeline</h1>
<p class="max-w-3xl text-sm text-muted-foreground">
The data workflows on GitHub Actions. Those marked data-write take turns: one runs, at most one waits, and a newer
one cancels the waiting run.
The data workflows on GitHub Actions. Those marked data-write take turns: one runs and the others wait in a queue,
first in, first out.
</p>
</div>
<Button variant="outline" size="sm" onclick={() => load(true)} disabled={loading}>
+65 -2
View File
@@ -1,7 +1,18 @@
/**
* DATA_WORKFLOWS must name every workflow in `concurrency: data-write`:
* auto-approve waits for exactly those, and one missing from the list is one
* whose pending run an automatic batch could evict.
* auto-approve waits for exactly those.
*
* And every one of them must:
* - queue (`queue: max`, `cancel-in-progress: false`). With the default a
* run that queued behind a running job evicted the one already waiting:
* Update Reviews lost five of ten runs that way in October 2026. Mixing
* the two in one group is not documented, so none may;
* - check out `main`. A run's commit is fixed when the run is created, and a
* run that waited behind another data job would write shards from before
* that job's commit - Auto Update JSON failed four days in ten on the
* rebase conflict in data/index.json;
* - declare `timeout-minutes`, and a JOB_BUDGET_MINUTES at least 20 below
* it (scripts/core/budget.py): the script stops in time to save and push.
*/
import { describe, expect, it } from "vitest";
import { existsSync, readdirSync, readFileSync } from "node:fs";
@@ -11,6 +22,26 @@ import { DATA_WORKFLOWS } from "./pipeline";
const REPO = join(__dirname, "..", "..");
const DIR = join(REPO, ".github", "workflows");
/** The YAML without its comment lines, which quote the old settings. */
function code(file: string): string {
return readFileSync(join(REPO, file), "utf-8")
.split("\n")
.filter((line) => !/^\s*#/.test(line))
.join("\n");
}
const TIMEOUT = /timeout-minutes\s*:\s*(.+)/;
const BUDGET = /JOB_BUDGET_MINUTES\s*:\s*(.+)/;
/** The whole numbers on the first line `key` matches: [60] for "60", [340, 45]
* for "${{ inputs.backfill && 340 || 45 }}". */
function numbers(text: string, key: RegExp): number[] {
const line = key.exec(text);
return line ? (line[1].match(/\d+/g) ?? []).map(Number) : [];
}
const dataWrite = DATA_WORKFLOWS.filter((w) => w.dataWrite).map((w) => w.file);
describe("DATA_WORKFLOWS", () => {
it("marks exactly the data-write workflows", () => {
const inGroup = readdirSync(DIR)
@@ -27,3 +58,35 @@ describe("DATA_WORKFLOWS", () => {
expect(new Set(DATA_WORKFLOWS.map((w) => w.file)).size).toBe(DATA_WORKFLOWS.length);
});
});
describe("every data-write workflow", () => {
it("queues its runs instead of evicting the one waiting", () => {
for (const file of dataWrite) {
const text = code(file);
expect(text, file).toMatch(/\bqueue\s*:\s*max\b/);
expect(text, file).toMatch(/\bcancel-in-progress\s*:\s*false\b/);
}
});
it("checks out main, not the commit its run was created on", () => {
for (const file of dataWrite) {
const text = code(file);
const checkouts = text.match(/uses\s*:\s*actions\/checkout@/g) ?? [];
const refs = text.match(/\bref\s*:\s*main\b/g) ?? [];
expect(checkouts.length, file).toBeGreaterThan(0);
expect(refs.length, file).toBe(checkouts.length);
}
});
it("stops at its budget in time to save before its timeout", () => {
for (const file of dataWrite) {
const text = code(file);
const timeouts = numbers(text, TIMEOUT);
const budgets = numbers(text, BUDGET);
expect(timeouts.length, file).toBeGreaterThan(0);
if (budgets.length === 0) continue;
expect(budgets.length, file).toBe(timeouts.length);
budgets.forEach((b, i) => expect(b + 20, `${file}: budget ${b}, timeout ${timeouts[i]}`).toBeLessThanOrEqual(timeouts[i]));
}
});
});
+3 -4
View File
@@ -3,9 +3,9 @@
* the GitHub API), so renaming a workflow does not break the page.
*
* `dataWrite` marks the workflows in `concurrency: {group: data-write}`.
* Auto-approve commits only while none of them is running or waiting: the
* group keeps ONE pending run, and the ingest an approval starts would evict a
* job already waiting (pipeline.test.ts holds this list to the YAML).
* Auto-approve commits only while none of them is running or waiting, so the
* ingest an approval starts can begin at once (pipeline.test.ts holds this
* list to the YAML, and every one of them to `queue: max`).
*
* Pure: no DOM, no Env, no I/O.
*/
@@ -24,7 +24,6 @@ export const DATA_WORKFLOWS: DataWorkflow[] = [
{ file: ".github/workflows/update-reviews.yml", label: "Update reviews", dataWrite: true },
{ file: ".github/workflows/top-online.yml", label: "Top online", dataWrite: true },
{ file: ".github/workflows/top-offline.yml", label: "Top offline", dataWrite: true },
{ file: ".github/workflows/check-dead-links.yml", label: "Check dead links", dataWrite: true },
{ file: ".github/workflows/mark-dead-games.yml", label: "Mark dead games", dataWrite: true },
{ file: ".github/workflows/purge-unhealthy.yml", label: "Purge unhealthy games", dataWrite: true },
{ file: ".github/workflows/snapshot-daily.yml", label: "Daily snapshot", dataWrite: true },
+4 -3
View File
@@ -17,9 +17,10 @@
* ingested. Checked again atomically inside the commit
* (requireEmptyQueue). Not "no row is approved": a network-skipped row
* stays approved for 12 hours and would stall everything that long;
* - no data-write workflow is running or waiting (GitHub Actions). That
* group keeps ONE pending run, and the ingest this commit starts would
* evict a job already waiting its turn;
* - no data-write workflow is running or waiting (GitHub Actions). The
* group queues its runs (queue: max), so the ingest this commit starts
* would no longer evict a job waiting its turn - it would wait behind it.
* The gate stays so that a batch lands when the ingest can start at once;
* - something is eligible: pending, free, app_type game, health ok, old
* enough, from an allowed source, never decided, its game not committed,
* not in data/blocklist.jsonl.