Merge pull request #209 from poli0981/feat/pipeline-headroom
v4.4.0 — The Headroom Edition: data jobs that scale with the catalogue
This commit is contained in:
57 files changed
+2339
-324
No files matched your search
@@ -1,19 +0,0 @@
|
||||
name : Check Dead Links
|
||||
on :
|
||||
schedule : [ { cron : '0 4 1,6,11,16,21,26 * *' } ]
|
||||
workflow_dispatch :
|
||||
permissions : { contents : write }
|
||||
concurrency :
|
||||
group : data-write
|
||||
cancel-in-progress : false
|
||||
jobs :
|
||||
check :
|
||||
runs-on : ubuntu-latest
|
||||
steps :
|
||||
-
|
||||
uses : actions/checkout@v6
|
||||
-
|
||||
uses : actions/setup-python@v6
|
||||
with : { python-version : '3.15' }
|
||||
-
|
||||
run : chmod +x ./bash/dead_links.sh && ./bash/dead_links.sh
|
||||
@@ -5,26 +5,31 @@ on :
|
||||
# fires the ingest on a detached HEAD, and dies with exit 128 on push.
|
||||
#
|
||||
# data/blocklist.jsonl is here too: this job also applies removals
|
||||
# (bash/ingest.sh runs scripts/remove_games.py first). A separate removal
|
||||
# workflow would share `data-write` with this one, and each would evict the
|
||||
# other's pending run.
|
||||
# (bash/ingest.sh runs scripts/remove_games.py first), so one run does both.
|
||||
push : { branches : [ main ], paths : [ 'scripts/temp_info.jsonl', 'data/blocklist.jsonl' ] }
|
||||
# Orphan recovery, not a poll. `data-write` keeps at most ONE run pending and
|
||||
# cancels the older one, so a push here that queues behind a ~115-minute data
|
||||
# job is silently evicted the moment any OTHER data-write workflow queues --
|
||||
# and nothing retries it, leaving approved games sitting in temp_info.jsonl
|
||||
# until the next unrelated push. Re-running on a clear file is free:
|
||||
# ingest_new.py returns immediately and `git diff --staged --quiet` skips the
|
||||
# commit, so this is idempotent by construction.
|
||||
# A safety net, not a poll. Before data-write queued its runs (queue: max),
|
||||
# a push here that waited behind a long data job was evicted the moment any
|
||||
# other data-write workflow queued, and approved games sat in
|
||||
# temp_info.jsonl until the next unrelated push. Nothing is evicted now, but
|
||||
# a failed run still leaves its links queued. Re-running on a clear file is
|
||||
# free: ingest_new.py returns immediately and `git diff --staged --quiet`
|
||||
# skips the commit, so this is idempotent by construction.
|
||||
schedule : [ { cron : '0 */3 * * *' } ]
|
||||
workflow_dispatch :
|
||||
permissions : { contents : write }
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency :
|
||||
group : data-write
|
||||
cancel-in-progress : false
|
||||
queue : max
|
||||
jobs :
|
||||
ingest :
|
||||
runs-on : ubuntu-latest
|
||||
# About seven seconds a game, and an /admin batch holds at most 100. No
|
||||
# time budget: ingest_new.py empties the whole queue file when it is done,
|
||||
# so a game it had not reached would be lost.
|
||||
timeout-minutes : 120
|
||||
steps :
|
||||
-
|
||||
uses : actions/checkout@v6
|
||||
|
||||
@@ -19,13 +19,17 @@ permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
jobs:
|
||||
mark:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# The tip of main when the job starts, not when it was queued: a run
|
||||
|
||||
@@ -13,7 +13,6 @@ on:
|
||||
- "Auto Update JSON Data"
|
||||
- "Top Online Leaderboard"
|
||||
- "Update Reviews"
|
||||
- "Check Dead Links"
|
||||
- "Purge Unhealthy Games"
|
||||
- "Ingest New Game Links"
|
||||
- "Force Re-fetch All (v2.1 Schema)"
|
||||
|
||||
@@ -1,20 +1,45 @@
|
||||
name : Purge Unhealthy Games
|
||||
|
||||
# Takes out the games that left the store, stopped being free or are not out
|
||||
# yet - only when two of Steam's answers agree: IStoreBrowseService/GetItems
|
||||
# screens the whole catalogue in a hundred-odd requests, and appdetails
|
||||
# confirms each game it flags (scripts/purge_unhealthy.py has the rules and
|
||||
# the guards). A few minutes, so it runs daily. It replaces both the old
|
||||
# purge (appdetails for every game, 5.3 hours) and Check Dead Links, a 5.5-hour
|
||||
# HEAD sweep that never found anything: Steam sends a delisted app's page to
|
||||
# the store front, which answers 200.
|
||||
|
||||
on :
|
||||
schedule : [ { cron : '0 5 * * 1' } ]
|
||||
schedule : [ { cron : '0 7 * * *' } ]
|
||||
workflow_dispatch :
|
||||
inputs :
|
||||
dry_run :
|
||||
description : 'Dry run: screen and confirm, remove nothing'
|
||||
type : boolean
|
||||
default : false
|
||||
permissions : { contents : write }
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency :
|
||||
group : data-write
|
||||
cancel-in-progress : false
|
||||
queue : max
|
||||
jobs :
|
||||
purge :
|
||||
runs-on : ubuntu-latest
|
||||
timeout-minutes : 60
|
||||
steps :
|
||||
-
|
||||
uses : actions/checkout@v6
|
||||
# main as it is when the job STARTS: a run that waited its turn must
|
||||
# work on what the job before it committed.
|
||||
with : { ref : main }
|
||||
-
|
||||
uses : actions/setup-python@v6
|
||||
with : { python-version : '3.15' }
|
||||
-
|
||||
run : chmod +x ./bash/purge.sh && ./bash/purge.sh
|
||||
env : { STEAM_API_KEY : '${{ secrets.STEAM_API_KEY }}' }
|
||||
env :
|
||||
STEAM_API_KEY : '${{ secrets.STEAM_API_KEY }}'
|
||||
JOB_BUDGET_MINUTES : '40'
|
||||
PURGE_DRY_RUN : ${{ inputs.dry_run }}
|
||||
@@ -1,16 +1,25 @@
|
||||
name : Force Re-fetch All (v2.1 Schema)
|
||||
|
||||
# ⚠ MANUAL ONLY – this re-fetches ALL games from Steam.
|
||||
# Takes a long time for large datasets (300+ games ≈ 15-30 min).
|
||||
# Creates a backup at data.jsonl.bak before starting.
|
||||
# ⚠ MANUAL ONLY – this re-fetches games from Steam from scratch, about six
|
||||
# seconds a game: far longer than one job may run once the catalogue is in
|
||||
# the thousands. Run it in PARTS: part "1/8" takes an eighth of the games
|
||||
# (crc32 slices, scripts/refetch_all.py), and the runs for 1/8 ... 8/8 can be
|
||||
# started together - the data-write group queues them (queue: max). A part cut
|
||||
# short by its budget keeps what it did; starting it again carries on with the
|
||||
# games it did not reach. Creates a backup at data.bak before starting.
|
||||
|
||||
on :
|
||||
workflow_dispatch :
|
||||
inputs :
|
||||
confirm :
|
||||
description : 'Type "yes" to confirm force re-fetch of ALL games'
|
||||
description : 'Type "yes" to confirm force re-fetch'
|
||||
required : true
|
||||
type : string
|
||||
part :
|
||||
description : 'Which part, K/N (1/8 ... 8/8 for the whole catalogue)'
|
||||
required : true
|
||||
default : '1/8'
|
||||
type : string
|
||||
|
||||
permissions :
|
||||
contents : write
|
||||
@@ -18,16 +27,21 @@ permissions :
|
||||
concurrency :
|
||||
group : data-write
|
||||
cancel-in-progress : false
|
||||
queue : max
|
||||
|
||||
jobs :
|
||||
refetch :
|
||||
if : ${{ github.event.inputs.confirm == 'yes' }}
|
||||
runs-on : ubuntu-latest
|
||||
timeout-minutes : 120
|
||||
timeout-minutes : 350
|
||||
steps :
|
||||
-
|
||||
name : Checkout repo
|
||||
uses : actions/checkout@v6
|
||||
# main as it is when the job STARTS: a part that waited its turn must
|
||||
# work on what the part before it committed.
|
||||
with :
|
||||
ref : main
|
||||
|
||||
-
|
||||
name : Setup Python
|
||||
@@ -36,9 +50,11 @@ jobs :
|
||||
python-version : '3.15'
|
||||
|
||||
-
|
||||
name : Force re-fetch all games
|
||||
name : Force re-fetch one part
|
||||
env :
|
||||
STEAM_API_KEY : ${{ secrets.STEAM_API_KEY }}
|
||||
REFETCH_PART : ${{ inputs.part }}
|
||||
JOB_BUDGET_MINUTES : '320'
|
||||
run : |
|
||||
chmod +x ./bash/refetch_all.sh
|
||||
./bash/refetch_all.sh
|
||||
@@ -1,16 +1,18 @@
|
||||
name: Refresh All-Time Peaks
|
||||
|
||||
# all_time_peak - the most players a game has had at once - from SteamCharts'
|
||||
# recorded peak, a thirtieth of the catalogue a day (~186 pages, 2-3 s apart:
|
||||
# about ten minutes), so every game is asked once a month. Every run also
|
||||
# raises the field from our own Steam API samples, which needs no requests.
|
||||
# scripts/refresh_peaks.py has the rules: the field only ever goes up, and a
|
||||
# refusal from SteamCharts ends the run early with a warning, not a failure.
|
||||
# recorded peak, a thirtieth of the catalogue a day (~330 pages, 2-3 s apart:
|
||||
# about a quarter of an hour), so every game is asked once a month. Past 400
|
||||
# pages a day the cycle grows to 60 days and more instead: never more often
|
||||
# than once a month a game. Every run also raises the field from our own
|
||||
# Steam API samples, which needs no requests. scripts/refresh_peaks.py has
|
||||
# the rules: the field only ever goes up, and a refusal from SteamCharts - or
|
||||
# the time budget running out - ends the run early with a warning, not a
|
||||
# failure.
|
||||
#
|
||||
# 19:30 UTC sits between Refresh Store Data (16:00, ~40 minutes) and the Daily
|
||||
# Snapshot (23:00), where only the short Ingest runs are due. Scheduled runs
|
||||
# here start up to a few hours late, and the data-write group keeps only ONE
|
||||
# pending run, so this stays clear of the long morning jobs.
|
||||
# 19:30 UTC sits between Refresh Store Data (16:00, about an hour) and the
|
||||
# Daily Snapshot (23:00). A data job still running only delays this one: the
|
||||
# data-write group queues runs (queue: max) instead of evicting them.
|
||||
#
|
||||
# The one-time backfill is a manual run with the box ticked: every game, about
|
||||
# four hours. Start it in the evening UTC, after Refresh Store Data.
|
||||
@@ -30,15 +32,20 @@ permissions: {contents: write}
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
jobs:
|
||||
refresh:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: ${{ inputs.backfill && 330 || 30 }}
|
||||
timeout-minutes: ${{ inputs.backfill && 340 || 45 }}
|
||||
steps:
|
||||
# main as it is when the job STARTS: a run that waited its turn must
|
||||
# work on what the job before it committed.
|
||||
- uses: actions/checkout@v6
|
||||
with: {ref: main}
|
||||
- uses: actions/setup-python@v6
|
||||
with: {python-version: '3.15'}
|
||||
- run: chmod +x ./bash/refresh_peaks.sh && ./bash/refresh_peaks.sh
|
||||
env:
|
||||
PEAKS_ALL: ${{ inputs.backfill }}
|
||||
JOB_BUDGET_MINUTES: ${{ inputs.backfill && '300' || '25' }}
|
||||
@@ -5,22 +5,23 @@ name: Refresh Store Data
|
||||
# store page, every game once a week; name, header image, description,
|
||||
# developer/publisher, release date, platforms, Metacritic and DRM notes from
|
||||
# appdetails, every game once a month. Each daily run takes one slice of each
|
||||
# rotation - roughly 630 store pages and 150 appdetails calls, ~40 minutes.
|
||||
# rotation - about 1,400 store pages and 330 appdetails calls at 9,800 games,
|
||||
# an hour. Past 2,000 pages or 500 appdetails a day a cycle doubles instead of
|
||||
# the run growing, so it stays under an hour and a half.
|
||||
# scripts/refresh_store_data.py explains the rotation and the rules (never
|
||||
# blanks a value, never touches MANUAL_FIELDS).
|
||||
#
|
||||
# Then the anti-cheat sweep, committed separately: every game's anti-cheat as
|
||||
# its developer declares it on Steam, about 160 GetItems requests plus a store
|
||||
# its developer declares it on Steam, about 200 GetItems requests plus a store
|
||||
# page for each disclosure Steam does not name there (~70), a few minutes.
|
||||
# scripts/refresh_anticheat.py has its rules: Steam wins, an override wins
|
||||
# over Steam, and no disclosure never clears a value. `only: anticheat` runs
|
||||
# just the sweep.
|
||||
#
|
||||
# 16:00 UTC is the quiet part of this repository's day. Scheduled runs here
|
||||
# start two to five hours late, the morning data jobs (reviews, dead links,
|
||||
# purge) run for two to four hours, and the data-write group keeps only ONE
|
||||
# pending run: a second one queueing behind a running job cancels the first.
|
||||
# Nothing else long is due this late; the Daily Snapshot (23:00) comes after.
|
||||
# 16:00 UTC is the quiet part of this repository's day: scheduled runs here
|
||||
# start up to an hour late, and the morning jobs are done by then. A data job
|
||||
# that is still running only delays this one - the data-write group queues
|
||||
# runs (queue: max) instead of evicting the one waiting.
|
||||
|
||||
on:
|
||||
schedule: [{cron: '0 16 * * *'}]
|
||||
@@ -38,11 +39,12 @@ permissions: {contents: write}
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
jobs:
|
||||
refresh:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 120
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
# main, not the triggering commit: a run that waited its turn in
|
||||
# data-write must work on the data as it is now, not as it was queued.
|
||||
@@ -53,3 +55,6 @@ jobs:
|
||||
- run: chmod +x ./bash/refresh_store.sh && ./bash/refresh_store.sh
|
||||
env:
|
||||
REFRESH_ONLY: ${{ inputs.only }}
|
||||
# The store refresh's budget (core/budget.py); the anti-cheat sweep
|
||||
# after it takes a few minutes and has none.
|
||||
JOB_BUDGET_MINUTES: '100'
|
||||
@@ -4,15 +4,22 @@ on:
|
||||
workflow_dispatch:
|
||||
permissions: {contents: write}
|
||||
# Shares the data-write group with every other workflow that commits to
|
||||
# data/ — without it this can push mid-run against a 2-hour job.
|
||||
# data/ — without it this can push mid-run against a long job. queue: max -
|
||||
# a run that finds another data job running waits its turn (first in, first
|
||||
# out) instead of evicting the run already waiting.
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
jobs:
|
||||
snapshot:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
# main as it is when the job STARTS, so the snapshot holds what the
|
||||
# jobs before it committed.
|
||||
- uses: actions/checkout@v4
|
||||
with: {ref: main}
|
||||
- uses: actions/setup-python@v5
|
||||
with: {python-version: '3.15'}
|
||||
- run: chmod +x ./bash/snapshot.sh && ./bash/snapshot.sh
|
||||
@@ -5,15 +5,27 @@ on:
|
||||
schedule: [{cron: '0 5 1,15 * *'}]
|
||||
workflow_dispatch:
|
||||
permissions: {contents: write}
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
jobs:
|
||||
leaderboard:
|
||||
runs-on: ubuntu-latest
|
||||
# About 0.55 s a game through the keyed Web API: ~65 minutes for 7,000
|
||||
# offline games, ~2 hours at 14,000. The budget stops the sampling with
|
||||
# time left to save and commit.
|
||||
timeout-minutes: 240
|
||||
steps:
|
||||
# main as it is when the job STARTS: a run that waited its turn must
|
||||
# work on what the job before it committed.
|
||||
- uses: actions/checkout@v4
|
||||
with: {ref: main}
|
||||
- uses: actions/setup-python@v5
|
||||
with: {python-version: '3.15'}
|
||||
- run: chmod +x ./bash/offline.sh && ./bash/offline.sh
|
||||
env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'}
|
||||
env:
|
||||
STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'
|
||||
JOB_BUDGET_MINUTES: '200'
|
||||
@@ -3,15 +3,27 @@ on:
|
||||
schedule: [{cron: '0 3 * * 0,3,5'}]
|
||||
workflow_dispatch:
|
||||
permissions: {contents: write}
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
jobs:
|
||||
leaderboard:
|
||||
runs-on: ubuntu-latest
|
||||
# The player counts come from the keyed Web API with no pause between
|
||||
# batches, about 0.55 s a game: ~27 minutes for 2,900 online games. The
|
||||
# budget stops the sampling with time left to save and commit.
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
# main as it is when the job STARTS: a run that waited its turn must
|
||||
# work on what the job before it committed.
|
||||
- uses: actions/checkout@v4
|
||||
with: {ref: main}
|
||||
- uses: actions/setup-python@v5
|
||||
with: {python-version: '3.15'}
|
||||
- run: chmod +x ./bash/online.sh && ./bash/online.sh
|
||||
env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'}
|
||||
env:
|
||||
STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'
|
||||
JOB_BUDGET_MINUTES: '120'
|
||||
@@ -3,15 +3,26 @@ on:
|
||||
schedule: [{cron: '0 0 * * *'}]
|
||||
workflow_dispatch:
|
||||
permissions: {contents: write}
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
jobs:
|
||||
update:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
# main as it is when the job STARTS. The commit the scheduled run was
|
||||
# created on is older whenever another data job ran first, and this one
|
||||
# then failed on a conflict in data/index.json (four times in ten days,
|
||||
# October 2026).
|
||||
- uses: actions/checkout@v6
|
||||
with: {ref: main}
|
||||
- uses: actions/setup-python@v6
|
||||
with: {python-version: '3.15'}
|
||||
- run: chmod +x ./bash/json.sh && ./bash/json.sh
|
||||
env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'}
|
||||
env:
|
||||
STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'
|
||||
JOB_BUDGET_MINUTES: '60'
|
||||
@@ -1,19 +1,40 @@
|
||||
name : Update Reviews
|
||||
|
||||
# Every game's review summary from IStoreBrowseService/GetItems, fifty games a
|
||||
# request: about two minutes for the whole catalogue (scripts/update_reviews.py
|
||||
# has the rules). It used to ask appreviews about one game at a time and, at
|
||||
# 9,800 games, ran into GitHub's six-hour limit with nothing saved.
|
||||
|
||||
on :
|
||||
schedule : [ { cron : '0 6 */2 * *' } ]
|
||||
schedule : [ { cron : '0 6 * * *' } ]
|
||||
workflow_dispatch :
|
||||
inputs :
|
||||
dry_run :
|
||||
description : 'Dry run: read Steam, write nothing'
|
||||
type : boolean
|
||||
default : false
|
||||
permissions : { contents : write }
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency :
|
||||
group : data-write
|
||||
cancel-in-progress : false
|
||||
queue : max
|
||||
jobs :
|
||||
reviews :
|
||||
runs-on : ubuntu-latest
|
||||
timeout-minutes : 60
|
||||
steps :
|
||||
-
|
||||
uses : actions/checkout@v6
|
||||
# main as it is when the job STARTS: a run that waited its turn must
|
||||
# work on what the job before it committed.
|
||||
with : { ref : main }
|
||||
-
|
||||
uses : actions/setup-python@v6
|
||||
with : { python-version : '3.15' }
|
||||
-
|
||||
run : chmod +x ./bash/reviews.sh && ./bash/reviews.sh
|
||||
env :
|
||||
JOB_BUDGET_MINUTES : '40'
|
||||
REVIEWS_DRY_RUN : ${{ inputs.dry_run }}
|
||||
@@ -13,8 +13,6 @@ Thumbs.db
|
||||
# scripts/mark_dead_games.py and explicitly staged by mark-dead-games.yml.
|
||||
# It survives *.log today only because ignore rules do not apply to
|
||||
# already-tracked files -- rename or recreate it and it silently vanishes.
|
||||
# (scripts/dead_links.log is deliberately NOT excepted: bash/dead_links.sh
|
||||
# runs `git add .`, and that log is local diagnostic output.)
|
||||
!scripts/dead_games.log
|
||||
*.tmp
|
||||
*.temp
|
||||
|
||||
@@ -4,6 +4,54 @@ All notable changes to this awesome noob repo will be documented here.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [v4.4.0] – 2026-10-23 (The "Headroom" Edition)
|
||||
|
||||
The data pipeline grows with the catalogue again. With almost 10,000 games
|
||||
and a few hundred more every day, the jobs that asked Steam about one game at
|
||||
a time had run out of time; they now read Steam's store API fifty games a
|
||||
request, wait their turn instead of cancelling each other, and stop in time to
|
||||
save. The release also carries the tooling merged since v4.3.3 - TypeScript
|
||||
7, Node 26 and Python 3.15, SonarCloud in CI, the desktop updater fix - and a
|
||||
certifi bump. The desktop and Android apps stay at `2.2.1`, with no new
|
||||
build. Repo public-facing version `4.3.3` → `4.4.0`.
|
||||
|
||||
### 🧰 Pipeline
|
||||
|
||||
- **The data jobs scale with the catalogue again.** With almost 10,000 games
|
||||
and a few hundred more every day, three jobs that asked Steam about one
|
||||
game at a time had reached GitHub's six-hour limit: on 2026-10-23 Update
|
||||
Reviews was stopped at exactly six hours with nothing saved. The full
|
||||
assessment, with every measurement, is `docs/plan/18-pipeline-scale.md`.
|
||||
- **Reviews come from Steam's store API, for every game, every day.** Fifty
|
||||
games a request, about two minutes for the catalogue, instead of every two
|
||||
days and five and a half hours. The percentage is now Steam's own, which
|
||||
rounds down: about 4,000 games show one point less once (Counter-Strike 2
|
||||
"85%" instead of "86%"), exactly as on their store pages. A summary that
|
||||
says 0 reviews is double-checked game by game before it replaces anything.
|
||||
- **The daily purge needs two answers to remove a game.** Steam's store API
|
||||
screens every game in a few minutes, and a game is removed only when
|
||||
appdetails agrees; "unavailable" is asked twice, five minutes apart. A run
|
||||
that would remove more than 100 games, or flags more than 500, stops before
|
||||
writing anything. It also lists earlier removals that Steam has since
|
||||
undone. It runs daily now, instead of a five-hour sweep on Mondays.
|
||||
- **Check Dead Links is retired.** In its whole life it never found a dead
|
||||
link: Steam sends a delisted game's page to the store front, which answers
|
||||
200. The purge's screen sees delisted games (`unlisted`) instead.
|
||||
- **No run is cancelled for waiting.** The data jobs share one queue that
|
||||
used to hold a single waiting run: a third job cancelled it, and Update
|
||||
Reviews lost five of ten runs that way. The queue now holds them all, first
|
||||
in, first out, and every job checks out the latest `main` when it starts -
|
||||
Auto Update JSON failed four days in ten from working on the data as it was
|
||||
when the run was queued.
|
||||
- **Every long job stops in time to save.** Each workflow gives its job a time
|
||||
budget below its timeout: past it the job saves and commits what it did
|
||||
instead of being stopped with nothing, and it warns before the budget gets
|
||||
tight. The leaderboards skip the pauses meant for store pages (Top Online
|
||||
~27 minutes instead of 48). The store-data and all-time-peak rotations
|
||||
stretch their cycle as the catalogue grows instead of their daily runs.
|
||||
- **Force Re-fetch runs in parts** (`1/8` … `8/8`, queued one after another),
|
||||
and a game whose fetch fails keeps its old data.
|
||||
|
||||
### 🧰 Tooling
|
||||
|
||||
- **Desktop releases keep reaching the updater.** tauri-action v1 writes each
|
||||
|
||||
@@ -222,7 +222,13 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
(tags, languages, `language_details`, `has_paid_dlc`) every 7 days,
|
||||
appdetails (name, header image, description, developer/publisher, release
|
||||
date, platforms, Metacritic, DRM notes) every 30. Never `appid % cycle`:
|
||||
appids are mostly multiples of 10, which leaves most slices empty. It
|
||||
appids are mostly multiples of 10, which leaves most slices empty. A cycle
|
||||
grows with the catalogue instead of the run (`rotation.cycle_for()`: past
|
||||
`PAGE_DAILY` 2,000 pages or `DETAILS_DAILY` 500 appdetails a day it becomes
|
||||
14, 21, ... or 60, 90, ...) and is always a MULTIPLE of its base, so a game
|
||||
keeps its weekday and its day of the month and visits only ever thin out.
|
||||
The slice is visited in `rotation.daily_order()`, so a run that meets its
|
||||
time budget leaves a different tail each day. It
|
||||
replaces values only with NON-EMPTY ones, never writes `MANUAL_FIELDS` or
|
||||
`notes`, trusts `has_paid_dlc` only from a real store page (an age gate has
|
||||
no language table or tags) or a gated game's DLC prices, compares every
|
||||
@@ -231,6 +237,9 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
counting as news), and bumps `last_updated` only on a real change. A game
|
||||
with empty tags, languages or `language_details` is in the page group EVERY
|
||||
day, first, capped at `MISSING_DAILY_CAP`; `--missing-only` runs just those.
|
||||
A game whose appdetails no longer call it free is LISTED in a `::warning::`,
|
||||
never written: purge removes a game only when the store API flags it too,
|
||||
and this is where the opposite case shows.
|
||||
A new field added to it must obey the same rules;
|
||||
`scripts/tests/test_refresh_store_data.py` holds them.
|
||||
- **A store page behind Steam's age gate is never parsed.** An 'Adult Only'
|
||||
@@ -252,7 +261,8 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
with `l=english`: without it, a runner was once answered in Russian.
|
||||
- **`all_time_peak` only ever goes UP, and `refresh_peaks.py` is what raises
|
||||
it** (daily 19:30 UTC, "Refresh All-Time Peaks"). A `crc32(appid) % 30`
|
||||
slice (`core/rotation.py`, shared with `refresh_store_data.py`) reads
|
||||
slice (`core/rotation.py`, shared with `refresh_store_data.py`; 60, 90, ...
|
||||
past `PEAK_DAILY` 400 pages a day, never shorter) reads
|
||||
SteamCharts' recorded all-time peak - the only source older than our own
|
||||
sampling: SteamDB forbids scraping and Valve's APIs know only today. Every
|
||||
run also raises it from our samples (`peak_today`, which despite its name is
|
||||
@@ -260,8 +270,9 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
`apply_players()` raises it on every sample, never from a 0. SteamCharts has
|
||||
no API, terms or robots.txt, so: one page per game a month, 2-3 s apart, a
|
||||
User-Agent that names the project; 500 is an app it does not track (most of
|
||||
the long tail), 404 one it has never seen; a 403, 429 or challenge page
|
||||
stops the run with a warning, not a failure. `refetch_all.py` must never
|
||||
the long tail), 404 one it has never seen; a 403, 429 or challenge page,
|
||||
or the job's time budget, stops the run with a warning, not a failure.
|
||||
`refetch_all.py` must never
|
||||
clear the field - it cannot be fetched again - and nothing may lower it.
|
||||
The backfill is a manual run with `backfill` ticked (~4 h, start it in the
|
||||
evening UTC). DISCLAIMER section 6 names the source, so rewording it reopens
|
||||
@@ -293,6 +304,42 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
name and the new key are the same family, and a known label stays as it was
|
||||
written (so "Easy Anti-Cheat" and "EAC" both occur; the site groups them).
|
||||
`scripts/tests/test_anticheat.py` and `test_refresh_anticheat.py` hold it.
|
||||
- **Reviews are Steam's own summary, read for EVERY game daily**
|
||||
(`update_reviews.py`, "Update Reviews" 06:00): `GetItems` with
|
||||
`include_reviews`, fifty games a request (`SteamClient.fetch_items()`, the
|
||||
sweep the anti-cheat and purge jobs share), ~2 minutes. It asked
|
||||
`appreviews` one game at a time until v4.4.0 and, at 9,800 games, ran into
|
||||
GitHub's six-hour limit with nothing saved. Rules: the percentage is Steam's,
|
||||
FLOORED (`reviews_from_summary()` floors appreviews in integers too, so an
|
||||
ingest and the refresh agree; until v4.4.0 it was rounded, one point higher
|
||||
on ~40% of games); a GetItems count of 0 is never written - its summary has
|
||||
said 0 for a game with thousands (845070) - so a stored percentage or "N/A"
|
||||
is asked on appreviews (`ASK_CAP` 300 a run, daily order) and only that
|
||||
answer is written; no item, no usable summary or a failed batch keeps the
|
||||
value. A record's `last_updated` moves on a change and on a HEARTBEAT: a
|
||||
game read whose stamp is 14-20 days old (crc32 jitter) is stamped again.
|
||||
The site calls a record older than 30 days stale (`web/src/lib/validation.ts`,
|
||||
`/health`), and this job used to stamp every game on every run; the
|
||||
heartbeat keeps "the pipeline confirmed this lately" true without rewriting
|
||||
every record daily. `scripts/tests/test_update_reviews.py` holds it.
|
||||
- **A game is purged only when two of Steam's answers agree**
|
||||
(`purge_unhealthy.py`, "Purge Unhealthy Games" daily 07:00). GetItems
|
||||
screens the whole catalogue (`store_items.screen_item()`: unknown to Steam,
|
||||
`visible` false, `unlisted`, `release.is_coming_soon`, no `is_free`; never
|
||||
`type` - a free mod is listed on purpose), and `check_game_health()`
|
||||
(appdetails) must confirm each flag. An `unavailable`/404 answer is asked
|
||||
again `RECHECK_AFTER` (5 min) later: appdetails answered `success:false` for
|
||||
14 live games in two bursts on 2026-09-27. A failed batch or a network error
|
||||
is never "gone"; blocklisted games are not asked about. More than 500 flags
|
||||
or 100 confirmed removals exits non-zero BEFORE anything is written
|
||||
(`--max-candidates`, `--max-removals` to override). Removals are logged to
|
||||
`removed_games.jsonl` exactly as before - `reconcile.ts` reads them. A
|
||||
`::notice::` names earlier removals GetItems now sees free, listed and
|
||||
released (re-queue the link to bring one back: a purged game keeps its D1
|
||||
decision and discovery never proposes it again). Check Dead Links is gone:
|
||||
HEAD on a delisted app's page ends on the store front with 200, so in its
|
||||
whole life it never found a dead link, and it took 5.5 hours.
|
||||
`scripts/tests/test_purge_unhealthy.py` holds it.
|
||||
- `ingest_new.py` only reads `scripts/temp_info.jsonl`, and a line is a store
|
||||
LINK and nothing else: every other key is ignored and the game is built from
|
||||
Steam (`merge_extension_data()` went with the archived browser extension;
|
||||
@@ -305,14 +352,15 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
appdetails calls other than `type: "game"` (`status_code: not_a_game` in
|
||||
`removed_games.jsonl`) - the check that makes auto-approve safe, and why a
|
||||
queued link cannot add a free DLC, demo or tool. The check lives in ingest
|
||||
only: in `health_checker` it would make Purge delete old records every week.
|
||||
only: in `health_checker` it would make Purge delete old records every day.
|
||||
- **`ingest-new.yml` is also the removal job.** `bash/ingest.sh` runs
|
||||
`remove_games.py` (applies `data/blocklist.jsonl`) and then
|
||||
`ingest_new.py`, and commits once ("Apply removals" when `temp_info` did not
|
||||
change). A separate removal workflow would keep evicting the ingest from
|
||||
`data-write`'s single pending slot, and the ingest it. The workflow checks
|
||||
out `ref: main`, not the triggering SHA: a run that waited its turn would
|
||||
otherwise work on the data as it was when it was queued.
|
||||
change), so one commit to either file starts one run that settles both.
|
||||
The workflow checks out `ref: main`, not the triggering SHA, like every
|
||||
data-write workflow: a run that waited its turn would otherwise work on the
|
||||
data as it was when it was queued. It has no time budget: `clear_temp()`
|
||||
empties the whole queue file, so a game it had not reached would be lost.
|
||||
- New games ARE discovered automatically now, by `scripts/discover_new.py`
|
||||
into the D1 review queue — but discovery only PROPOSES. Publication is
|
||||
decided in `/admin`: by a reviewer, batch by batch, or by the auto-approve
|
||||
@@ -329,8 +377,10 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
page previews is the SQL that runs). The empty-file gate is checked again
|
||||
inside the commit (`requireEmptyQueue`), so a batch never lands mid-ingest;
|
||||
"no row is approved" would have stalled it for the 12 hours a skipped row
|
||||
stays approved. The idle gate exists because a push while a data job waits
|
||||
EVICTS that job. `shared/pipeline.ts` lists every `data-write` workflow by
|
||||
stays approved. The idle gate dates from when a push while a data job waited
|
||||
EVICTED that job; with `queue: max` the ingest would only queue behind it,
|
||||
and the gate stays so a batch lands when its ingest can start at once.
|
||||
`shared/pipeline.ts` lists every `data-write` workflow by
|
||||
file, and `pipeline.test.ts` fails when a workflow joins the group without
|
||||
joining the list. Settings, limits and their validation are
|
||||
`shared/automation.ts`, imported by the Worker and the page. A scheduled
|
||||
@@ -393,16 +443,40 @@ scraping pipeline (`scripts/`) driven by GitHub Actions.
|
||||
(wrangler.jsonc, 180). `docs/PRIVACY_POLICY.md` states the number: change
|
||||
both together. `ingest_queue` and `ingest_decisions` are never pruned — the
|
||||
decisions are what keep a rejected game out of the discovery sweep.
|
||||
- Every workflow that writes to `data/` shares `concurrency: {group: data-write}`.
|
||||
A new data-writing workflow without it will race jobs that run ~2 hours.
|
||||
The group holds ONE pending run - a second queueing run cancels the first -
|
||||
and scheduled runs here start 2-5 hours after their cron, so a new schedule
|
||||
goes where nothing long is running (the long jobs - reviews, dead links,
|
||||
purge - start mid-morning UTC and run 2-4 hours; Refresh Store Data is at
|
||||
16:00 for that reason).
|
||||
- **Every workflow that writes to `data/` shares `concurrency: {group:
|
||||
data-write, cancel-in-progress: false, queue: max}`, checks out `ref: main`,
|
||||
and declares `timeout-minutes`.** `pipeline.test.ts` fails on a data-write
|
||||
workflow without any of them. A new data-writing workflow outside the group
|
||||
would race the others.
|
||||
- `queue: max` (GitHub, 2026-05-07) keeps up to 100 runs waiting, first in,
|
||||
first out. The default keeps ONE: a run queueing behind a running job
|
||||
cancelled the one already waiting - Update Reviews lost five of ten runs
|
||||
that way in October 2026. Mixing the two settings in one group is
|
||||
undocumented, so every member has `queue: max`.
|
||||
- `ref: main`: a run's commit is fixed when the run is CREATED. A scheduled
|
||||
run that waited behind another data job wrote shards from before that
|
||||
job's commit, and `commit_push.sh`'s rebase then conflicted on
|
||||
`data/index.json` - Auto Update JSON failed four days in ten that way.
|
||||
- Scheduled runs start up to an hour after their cron (2-5 hours in
|
||||
September). Waiting costs nothing now, but a long new job still goes where
|
||||
nothing long runs (Refresh Store Data is at 16:00 for that reason).
|
||||
- **Every long loop has a time budget** (`scripts/core/budget.py`). The
|
||||
workflow sets `JOB_BUDGET_MINUTES` at least 20 below its `timeout-minutes`
|
||||
(`pipeline.test.ts` checks); past it a loop starts no further game, the
|
||||
script saves what it did and the job commits it, and a loop that used more
|
||||
than three quarters says so in a `::warning::`. Loops visit their games in
|
||||
`rotation.daily_order()`, so a cut falls elsewhere the next day. A job
|
||||
stopped at its timeout commits nothing and concludes `cancelled`, which
|
||||
`notify-ci-failure.yml` (a reusable workflow in `poli0981/.github`) does not
|
||||
report: the budget is what keeps a growing catalogue from failing silently.
|
||||
`process_batch(pause=False)` for the keyed Web API (player counts); the
|
||||
15-30 s rest between batches stays for store pages and appdetails. Never a
|
||||
budget for `ingest_new.py`. `refetch_all.py` runs in parts (`--part K/N`)
|
||||
and clears each game on a copy that replaces it only once appdetails has
|
||||
answered, so a cut part leaves every other game as it was.
|
||||
- **Every data job commits through `bash/commit_push.sh`.** The group
|
||||
serialises workflows, not `main`: a merged PR or an /admin commit can land
|
||||
during a four-hour job, and a bare `git push` is then rejected and the run
|
||||
during a long job, and a bare `git push` is then rejected and the run
|
||||
thrown away (purge-unhealthy, 2026-09-21). The helper rebases onto
|
||||
`origin/main` and retries; none of those writers touch the shards, so the
|
||||
rebase is clean. A new wrapper must use it, once per commit (Refresh Store
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
[](https://free-steam-games.win/)
|
||||
[](https://github.com/poli0981/free-steam-games-list/releases)
|
||||
[](https://github.com/poli0981/free-steam-games-list/releases)
|
||||
[](.github/workflows/purge-unhealthy.yml)
|
||||
[](.github/workflows/purge-unhealthy.yml)
|
||||

|
||||
[](.github/FUNDING.yml)
|
||||
|
||||
@@ -89,8 +89,8 @@ The Python pipeline that powered v2 is untouched and runs daily as before:
|
||||
|
||||
- Schema v2.2 (23 + manual fields per record), CC BY 4.0 (see [LICENSE-DATA](LICENSE-DATA)), sharded data in `data/data_*.jsonl` (cap 800 records/shard) plus a `data/index.json` manifest with a SHA-256 per shard.
|
||||
- HTML scraper for accurate language tables, DLC pricing, user tags and the developer's anti-cheat disclosure — single GET per game.
|
||||
- Health-checked: dead-link sweep every 5 days, full purge every Monday.
|
||||
- Reviews-only refresh every 2 days; full refresh daily.
|
||||
- Health-checked daily: Steam's store API screens every game, and a game is removed only when appdetails agrees.
|
||||
- Reviews refreshed daily from Steam's own summaries; full refresh daily. Every data job queues for its turn and has a time budget, so a growing catalogue slows a refresh down instead of breaking it ([plan](docs/plan/18-pipeline-scale.md)).
|
||||
|
||||
</details>
|
||||
|
||||
@@ -151,7 +151,8 @@ scripts/
|
||||
│ ├── anticheat.py # Steam's anti-cheat disclosure and the rule that applies it
|
||||
│ ├── steamcharts.py # SteamCharts all-time peak reader (polite, stops on refusal)
|
||||
│ ├── peaks.py # all_time_peak: only ever raised
|
||||
│ ├── rotation.py # crc32 daily slices shared by the refresh jobs
|
||||
│ ├── rotation.py # crc32 daily slices shared by the refresh jobs, growing with the catalogue
|
||||
│ ├── budget.py # A job's time budget: stop in time to save, warn before it runs out
|
||||
│ └── health_checker.py # Game status detection with cached API data
|
||||
├── discover_new.py # Propose new games into the /admin review queue
|
||||
├── ingest_new.py # Add queued games (health check + fetch from Steam)
|
||||
@@ -160,13 +161,12 @@ scripts/
|
||||
├── refresh_store_data.py # Daily rotation: tags/languages weekly, store details monthly
|
||||
├── refresh_anticheat.py # Daily: every game's anti-cheat, as its developer declares it on Steam
|
||||
├── refresh_peaks.py # Daily rotation: all-time peak from SteamCharts, monthly per game
|
||||
├── update_reviews.py # Reviews-only (every 2 days)
|
||||
├── update_reviews.py # Daily: every game's review summary from Steam, ~2 minutes
|
||||
├── mark_dead_games.py # Flag online games with no players
|
||||
├── check_dead_links.py # HEAD-request 404/410 scanner
|
||||
├── purge_unhealthy.py # Remove delisted/not-free/invalid
|
||||
├── purge_unhealthy.py # Daily: remove delisted/not-free/unreleased (store API screen + appdetails)
|
||||
├── top_online.py / top_offline.py # Leaderboards
|
||||
├── edit_game.py # Overrides from the command line
|
||||
├── refetch_all.py # Force re-fetch ALL (manual only)
|
||||
├── refetch_all.py # Force re-fetch, in parts (manual only)
|
||||
└── delete_game.py # Remove games (blocklist) from the command line
|
||||
```
|
||||
|
||||
@@ -181,14 +181,13 @@ scripts/
|
||||
| Ingest New | On `scripts/temp_info.jsonl` or `data/blocklist.jsonl` push, and every 3 h | Apply removals, then add approved games |
|
||||
| Top Online | Sun/Wed/Fri 03:00 UTC | Live player leaderboard |
|
||||
| Top Offline | 1st + 15th of month 05:00 UTC | Leaderboard for single-player F2P concurrents |
|
||||
| Update Reviews | Every 2 days 06:00 UTC | Reviews-only refresh |
|
||||
| Update Reviews | Daily 06:00 UTC | Every game's review summary from Steam's store API |
|
||||
| Refresh Store Data | Daily 16:00 UTC | Rotating re-fetch: tags, languages, DLC flag weekly; name, image, description, developer/publisher, release date, platforms, Metacritic, DRM monthly. Then every game's anti-cheat from Steam's disclosure (committed separately; manual run with `only: anticheat` does just that) |
|
||||
| Refresh All-Time Peaks | Daily 19:30 UTC | `all_time_peak` from SteamCharts, every game once a month, plus our own samples; manual run with `backfill` asks about every game |
|
||||
| Check Dead Links | Every 5 days 04:00 UTC | HEAD-request 404/410 scanner |
|
||||
| Purge Unhealthy | Weekly Mon 05:00 UTC | Full health scan + removal |
|
||||
| Purge Unhealthy | Daily 07:00 UTC | Store API screen of every game; removal once appdetails agrees |
|
||||
| Mark Dead Games | Mon + Thu 04:30 UTC | Online + >1y + 0 players ≥14d → `is_dead=true` + 💀 note |
|
||||
| Daily Snapshot | Daily 23:00 UTC | Per-game players and reviews into `data/snapshots/` |
|
||||
| Force Re-fetch | Manual only | Re-fetch ALL games |
|
||||
| Force Re-fetch | Manual only | Re-fetch games from scratch, one part (`1/8` … `8/8`) a run |
|
||||
| Web CI / Python Tests | On `web/**` / `scripts/**` change | Type check, tests, build and output checks |
|
||||
| Release Desktop / Android | On `desktop-v*` / `android-v*` tag | Build the apps into a draft release |
|
||||
| Announce Discussion | On release `published` | Auto-create a GitHub Discussion in `Announcements` |
|
||||
|
||||
+6
-7
@@ -1,13 +1,12 @@
|
||||
#!/bin/bash
|
||||
# Commit what a data job changed, and push it even if main moved meanwhile.
|
||||
#
|
||||
# A data job can run for hours (update-reviews, check-dead-links,
|
||||
# purge-unhealthy), and `concurrency: data-write` serialises only the
|
||||
# WORKFLOWS. main still moves underneath one: a merged pull request, an /admin
|
||||
# approval, removal or override the Worker commits. A plain `git push` is then
|
||||
# rejected ("fetch first")
|
||||
# and the whole run is thrown away - purge-unhealthy lost four hours of work
|
||||
# that way on 2026-09-21.
|
||||
# A data job can run for an hour or more (refresh-store-data, the
|
||||
# leaderboards), and `concurrency: data-write` serialises only the WORKFLOWS.
|
||||
# main still moves underneath one: a merged pull request, an /admin approval,
|
||||
# removal or override the Worker commits. A plain `git push` is then rejected
|
||||
# ("fetch first") and the whole run is thrown away - purge-unhealthy lost four
|
||||
# hours of work that way on 2026-09-21.
|
||||
#
|
||||
# None of those other writers touch the shards, so rebasing onto them and
|
||||
# pushing again is safe. A genuine conflict still fails the job, loudly.
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
|
||||
python scripts/check_dead_links.py
|
||||
bash "$(dirname "$0")/commit_push.sh" "Dead link check [$(date +'%Y-%m-%d')]"
|
||||
+4
-3
@@ -3,9 +3,10 @@
|
||||
#
|
||||
# remove_games.py applies data/blocklist.jsonl (games the maintainer removed in
|
||||
# /admin or with delete_game.py) before ingest_new.py adds the queued links.
|
||||
# One workflow does both on purpose: `data-write` keeps ONE pending run and a
|
||||
# newer one evicts it, so a separate removal workflow and the ingest would keep
|
||||
# cancelling each other. Whichever run survives does both.
|
||||
# One workflow does both: a commit to either file starts one run that settles
|
||||
# the two, in that order. (It was also the only way to do it while
|
||||
# `data-write` kept a single pending run and a separate removal workflow and
|
||||
# the ingest kept evicting each other; the group queues them now.)
|
||||
set -euo pipefail
|
||||
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
|
||||
python scripts/remove_games.py
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
|
||||
# PURGE_DRY_RUN is the manual run's "dry run" box: screen and confirm, remove nothing.
|
||||
# A guard that trips (too many flagged or confirmed) exits non-zero before
|
||||
# anything is written, so set -e stops here and nothing is committed.
|
||||
if [[ "${PURGE_DRY_RUN:-}" == "true" ]]; then
|
||||
python scripts/purge_unhealthy.py --dry-run
|
||||
exit 0
|
||||
fi
|
||||
python scripts/purge_unhealthy.py
|
||||
bash "$(dirname "$0")/commit_push.sh" "Purge unhealthy [$(date +'%Y-%m-%d')]"
|
||||
+6
-3
@@ -1,7 +1,10 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
echo "── Force re-fetch ALL games (v2.1 schema) ──"
|
||||
# REFETCH_PART is the manual run's "part" box, K/N (scripts/refetch_all.py
|
||||
# checks it): the whole catalogue no longer fits in one job.
|
||||
part="${REFETCH_PART:-1/1}"
|
||||
echo "── Force re-fetch, part ${part} ──"
|
||||
|
||||
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
|
||||
python scripts/refetch_all.py
|
||||
bash "$(dirname "$0")/commit_push.sh" "Force re-fetch all games → v2.1 schema [$(date +'%Y-%m-%d')]"
|
||||
python scripts/refetch_all.py --part "$part"
|
||||
bash "$(dirname "$0")/commit_push.sh" "Force re-fetch, part ${part} [$(date +'%Y-%m-%d')]"
|
||||
@@ -1,5 +1,10 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
|
||||
# REVIEWS_DRY_RUN is the manual run's "dry run" box: read Steam, write nothing.
|
||||
if [[ "${REVIEWS_DRY_RUN:-}" == "true" ]]; then
|
||||
python scripts/update_reviews.py --dry-run
|
||||
exit 0
|
||||
fi
|
||||
python scripts/update_reviews.py
|
||||
bash "$(dirname "$0")/commit_push.sh" "Update reviews [$(date +'%Y-%m-%d')]"
|
||||
+5
-4
@@ -659,10 +659,11 @@ counts as *no* — the page's *Last run* line says which one stopped it:
|
||||
- **The previous batch has been ingested**: `scripts/temp_info.jsonl` on `main`
|
||||
is empty. Checked again inside the commit itself, so a batch never lands in
|
||||
the middle of an ingest.
|
||||
- **No data job is running or waiting.** The `data-write` group keeps one
|
||||
waiting run, and the ingest a batch starts would cancel a job already waiting
|
||||
its turn. The list of those workflows is `web/shared/pipeline.ts`, which a
|
||||
test holds to the YAML.
|
||||
- **No data job is running or waiting.** The `data-write` group queues its
|
||||
runs (`queue: max`), so the ingest a batch starts would only wait behind such
|
||||
a job; the gate makes a batch land when its ingest can start at once. The
|
||||
list of those workflows is `web/shared/pipeline.ts`, which a test holds to
|
||||
the YAML.
|
||||
- **Today's budget is not spent.** It is reserved before the commit and given
|
||||
back if the commit fails.
|
||||
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
# Pipeline scale — the data workflows as the catalogue grows
|
||||
|
||||
## Why this note exists
|
||||
|
||||
On 2026-10-23 the maintainer asked for an assessment of the GitHub Actions
|
||||
data workflows now that the catalogue grows by hundreds of games a day, a
|
||||
direction for them, and the change. This is that assessment and the decisions
|
||||
it led to. Every number below was measured on the day (`main` at 01a631bb,
|
||||
9,847 games), with `gh run list` over each workflow's last 40 runs and live,
|
||||
read-only requests to Steam.
|
||||
|
||||
The short version: three jobs asked Steam about every game one request at a
|
||||
time, two seconds a game, and had reached GitHub's six-hour limit; the
|
||||
`data-write` group cancelled half of the runs that waited for one of them;
|
||||
and the runs that did wait worked on stale data. Steam answers fifty games a
|
||||
request in its store API, and GitHub can now queue runs. The pipeline uses
|
||||
both, and every long loop now stops in time to save.
|
||||
|
||||
## What the catalogue does to each job
|
||||
|
||||
The catalogue held 9,847 games and grew by about 280 a day (auto-approve is
|
||||
capped at 500); Steam lists about 17,000 free games matching its filters, so
|
||||
the plan is sized for 20,000.
|
||||
|
||||
A per-game loop cost about 2.05 s a game against store endpoints (a
|
||||
1.2-2.0 s throttle, plus `process_batch`'s 15-30 s rest after every 50 games)
|
||||
and about 1.0 s against the keyed Web API, of which the rests were half.
|
||||
|
||||
| Workflow | Before | Measured | At 20,000 games |
|
||||
| --- | --- | --- | --- |
|
||||
| Update Reviews (every 2 days) | `appreviews`, one game a request | 2026-10-23: stopped at exactly 6:00:00, nothing saved; 5 of the last 10 runs cancelled while waiting | impossible |
|
||||
| Check Dead Links (every 5 days) | HEAD on every store page, flags 404/410 | 5.5 h on 2026-10-21; `dead_links.log` never existed and every game is `active`: Steam sends a delisted app's page to the store front, 200 | impossible |
|
||||
| Purge Unhealthy (Mondays) | appdetails for every game, one answer enough to remove | 5.3 h on 2026-10-19 | impossible |
|
||||
| Auto Update JSON (daily) | incomplete records only | 1 minute, but failed 4 days in 10 (below) | fine |
|
||||
| Top Online (3 a week) | player count for each online game | 48 min (2,898 games) | ~1.7 h |
|
||||
| Top Offline (twice a month) | player count for each offline game | 88 min (6,949 games) | ~4 h |
|
||||
| Refresh Store Data (daily) | a 7-day / 30-day rotation | 65 min of a 120-minute timeout | ~2.2 h: over |
|
||||
| Refresh All-Time Peaks (daily) | a 30-day rotation on SteamCharts | 14 min of a 30-minute timeout | ~29 min: at the edge |
|
||||
| Force Re-fetch (manual) | everything in one run | ~6.5 s a game, 18 hours | impossible |
|
||||
|
||||
Three problems were not about speed:
|
||||
|
||||
- **A stopped job is silent.** A job killed at its timeout commits nothing and
|
||||
concludes `cancelled`. `notify-ci-failure.yml` hands runs to a reusable
|
||||
workflow in `poli0981/.github` that reports only `failure`, so nobody heard
|
||||
that Update Reviews had stopped working.
|
||||
- **The `data-write` group kept ONE pending run.** A run that queued behind a
|
||||
running job cancelled the one already waiting. The ingest's three-hourly
|
||||
schedule made that happen every morning: Update Reviews was cancelled on
|
||||
10-05, 10-11, 10-15, 10-19 and 10-21.
|
||||
- **Queued runs worked on stale data.** A run's commit is fixed when the run
|
||||
is created. Auto Update JSON is created at 00:00 together with the ingest's
|
||||
schedule; when the ingest ran first and committed, Auto Update JSON wrote
|
||||
shards from before that commit, and `commit_push.sh`'s rebase failed on
|
||||
`data/index.json` (10-15, 10-16, 10-17, 10-18). Only three workflows checked
|
||||
out `ref: main`.
|
||||
|
||||
## What Steam's store API answers
|
||||
|
||||
`IStoreBrowseService/GetItems` (keyless, already used for gated games and the
|
||||
anti-cheat sweep) answered for the whole catalogue in 99 requests and 105
|
||||
seconds (100 ids a request; 200 also worked, 500 is refused at the edge for
|
||||
the URL's length). Each item carries what the slow jobs asked for:
|
||||
|
||||
- `reviews.summary_filtered`: `review_count`, `percent_positive`,
|
||||
`review_score_label` - the summary the store shows, in the vocabulary
|
||||
`appreviews` uses ("3 user reviews", "Mixed").
|
||||
- `visible`, `unlisted` (true for delisted apps), `is_free` (absent when not
|
||||
free), `release.is_coming_soon`, `type`.
|
||||
|
||||
Compared with the stored reviews (read from `appreviews` on 10-17, rounded):
|
||||
5,296 identical, 3,795 exactly one point lower - Steam floors its percentage
|
||||
(85 for Counter-Strike 2, which `round()` had made 86) - 287 with a new label,
|
||||
and 4 where GetItems said 0 reviews for a game the store page shows with
|
||||
thousands (845070) or a few. Its 0 is not to be trusted alone.
|
||||
|
||||
Screening every game the way purge would: 3 `unlisted` (appdetails calls them
|
||||
not free: removed), 5 without `is_free` (appdetails calls two of them free:
|
||||
kept), nothing hidden, unknown or coming soon. The dry run of the new purge on
|
||||
the real catalogue matched: 8 flagged, 3 removed, 5 kept by appdetails.
|
||||
|
||||
Of the 142 games removed before, GetItems now sees 27 as free and listed
|
||||
again. Most were never really gone: 14 were rejected by the ingest in two
|
||||
bursts on 2026-09-27, when appdetails answered `success:false` for live games,
|
||||
and were approved again later. Eight are still out; the new purge names them.
|
||||
|
||||
GitHub Actions has supported `concurrency.queue: max` since 2026-05-07: up to
|
||||
100 waiting runs, first in, first out, with `cancel-in-progress: false`.
|
||||
|
||||
## The direction
|
||||
|
||||
1. **Queue, don't evict.** Every data-write workflow has `queue: max` and
|
||||
checks out `ref: main`, so a run that waits loses nothing and works on what
|
||||
the run before it committed.
|
||||
2. **Batch, don't crawl.** Reviews and the purge screen come from GetItems:
|
||||
about 200 requests for the catalogue, whatever its size within reason.
|
||||
Requests per game are kept for confirming a removal and for the rare
|
||||
review count of 0.
|
||||
3. **Bound every run.** A job sets `JOB_BUDGET_MINUTES` below its timeout;
|
||||
a loop past it starts no further game and the script saves what it did.
|
||||
Loops visit games in an order that changes daily, rotations grow their
|
||||
cycle (a multiple of the base) instead of their daily slice, and a
|
||||
`::warning::` says when a loop used more than three quarters of its budget.
|
||||
4. **Drop what does nothing.** Check Dead Links goes, and so do the rests
|
||||
between batches of the keyed Web API.
|
||||
|
||||
Considered and left: a matrix of runners per job (still a request per game,
|
||||
parts racing each other's pushes, and SonarCloud's S8233 on workflow-level
|
||||
write permissions with more than one job), a self-hosted runner, the Worker's
|
||||
cron (the Worker may never write a shard), and a second opinion from GetItems
|
||||
for the ingest's rejections (a skipped link and a rejected one both end as a
|
||||
`failed` queue row; only the public log line differs).
|
||||
|
||||
## What changed
|
||||
|
||||
| Job | Now | Time at 9,847 | At 20,000 |
|
||||
| --- | --- | --- | --- |
|
||||
| Update Reviews | daily; GetItems, `appreviews` only for a 0 (at most 300) | ~2-5 min | ~5 min |
|
||||
| Purge Unhealthy | daily; GetItems screen, appdetails confirms each flag | ~3-8 min | ~5-10 min |
|
||||
| Check Dead Links | retired | 0 | 0 |
|
||||
| Top Online / Offline | no rests between batches | ~27 / ~64 min | ~55 min / ~2.1 h |
|
||||
| Refresh Store Data | cycles 7 → 14 past 2,000 pages a day, 30 → 60 past 500 appdetails | ~65 min | ~65-90 min |
|
||||
| Refresh All-Time Peaks | cycle 30 → 60 past 400 pages a day | ~14 min | ~15 min |
|
||||
| Force Re-fetch | parts `1/8` … `8/8`, queued | ~2.2 h a part | ~2.2 h a part (`1/16`…) |
|
||||
|
||||
Rules worth keeping (CLAUDE.md has them in full):
|
||||
|
||||
- **Reviews.** Steam's floored percentage, in integers (`29/100*100` is 28.99…
|
||||
in floating point). A GetItems count of 0 never writes; a stored percentage
|
||||
or "N/A" is asked on `appreviews` instead. No answer keeps the value.
|
||||
`last_updated` moves on a change, and on a heartbeat every 14-20 days for a
|
||||
game that was read: the site's "stale" badge (30 days) used to be kept quiet
|
||||
by this job stamping every game every run, which also rewrote every shard.
|
||||
- **Purge.** Remove only when GetItems flags a game AND `check_game_health()`
|
||||
agrees; ask an `unavailable` again five minutes later; never treat a failed
|
||||
batch or a network error as "gone"; stop before writing anything past 500
|
||||
flags or 100 removals; log as before; name earlier removals Steam has since
|
||||
undone. A purged game keeps its D1 decision, so discovery never proposes it
|
||||
again - the notice is the way back.
|
||||
- **Budgets.** Never for the ingest: `clear_temp()` empties its whole queue.
|
||||
Refetch clears a game on a copy and swaps it in only once appdetails has
|
||||
answered, oldest first, so a part cut short resumes where it stopped.
|
||||
|
||||
`web/shared/pipeline.test.ts` now fails on a data-write workflow without
|
||||
`queue: max`, `cancel-in-progress: false`, `ref: main` on every checkout, a
|
||||
`timeout-minutes`, or a budget less than 20 minutes below it.
|
||||
|
||||
## Load, before and after
|
||||
|
||||
| | Before | After |
|
||||
| --- | --- | --- |
|
||||
| Time the data-write group is busy | ~46 h a week | ~12 h a week |
|
||||
| Requests to Steam | ~82,000 a week | ~25,000 a week |
|
||||
| Runs cancelled while waiting | about one a day | none |
|
||||
|
||||
## Left for later
|
||||
|
||||
- **A timed-out run is still silent.** The reusable notifier in
|
||||
`poli0981/.github` reports `failure` only; reporting `cancelled` for the
|
||||
data workflows (now that waiting no longer cancels anything) belongs there.
|
||||
- **Workers Builds deploys on data commits** (`docs/DEPLOYMENT.md`, "Which
|
||||
commits deploy"). Daily reviews and purges add about one deploy a day until
|
||||
the watch paths are set.
|
||||
- **`data/snapshots/`** grows by 1.9 MB a day and nothing in the repository
|
||||
reads it. A leaner format (no name, no zero rows) would quarter it.
|
||||
- **The keyed player counts** are the last per-game loop. At 20,000 games
|
||||
Top Offline needs about two hours; past that, sample games with no players
|
||||
on a rotation instead of every run.
|
||||
- **Re-queue what Steam has undone**: the purge's notice lists the removed
|
||||
games that are free and on the store again.
|
||||
@@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dead link checker – HEAD requests only."""
|
||||
import sys, os, random, time
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from datetime import datetime, timezone
|
||||
from core.constants import DEAD_LINKS_LOG, BATCH_SIZE, BATCH_PAUSE_MIN, BATCH_PAUSE_MAX
|
||||
from core.data_store import load_main, save_main, extract_appid, now_iso, append_note_idempotent
|
||||
from core.steam_client import get_client
|
||||
|
||||
def main():
|
||||
games = load_main()
|
||||
active = [g for g in games if g.get("status", "active") == "active"]
|
||||
print(f"Checking {len(active)} games...\n")
|
||||
client, dead, total = get_client(), [], len(active)
|
||||
for bs in range(0, total, BATCH_SIZE):
|
||||
be = min(bs + BATCH_SIZE, total)
|
||||
print(f"── Batch {bs//BATCH_SIZE+1} ──")
|
||||
for i in range(bs, be):
|
||||
g = active[i]
|
||||
appid = extract_appid(g.get("link", ""))
|
||||
if not appid: continue
|
||||
name = g.get("name", appid)
|
||||
code = client.check_store_page(appid)
|
||||
if code in (404, 410):
|
||||
print(f" [{i+1}/{total}] 💀 {name} ({code})")
|
||||
g["status"] = "delisted"
|
||||
append_note_idempotent(g, "💀 Delisted", f"💀 Delisted ({code})")
|
||||
g["last_updated"] = now_iso()
|
||||
dead.append({"name": name, "appid": appid, "code": code})
|
||||
else:
|
||||
print(f" [{i+1}/{total}] {'✓' if code==200 else f'⚠{code}'} {name[:50]}")
|
||||
if be < total:
|
||||
time.sleep(random.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX))
|
||||
save_main(games)
|
||||
if dead:
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
with open(DEAD_LINKS_LOG, "a", encoding="utf-8") as f:
|
||||
f.write(f"\n── {ts} ──\n")
|
||||
for d in dead: f.write(f" [{d['code']}] {d['name']} ({d['appid']})\n")
|
||||
print(f"\n💀 {len(dead)} dead")
|
||||
else:
|
||||
print(f"\n✓ All {total} alive!")
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
@@ -0,0 +1,81 @@
|
||||
"""
|
||||
How long a data job may keep starting work, so that a catalogue which outgrows
|
||||
its schedule makes a run partial instead of making it fail.
|
||||
|
||||
A GitHub-hosted job is stopped at its timeout-minutes (six hours at most), and
|
||||
a stopped job commits nothing: Update Reviews ran for exactly six hours on
|
||||
2026-10-23 and threw away every review it had read. Each data workflow sets
|
||||
JOB_BUDGET_MINUTES at least twenty minutes below its timeout. A loop that
|
||||
reaches the budget starts no further game, the script saves what it did, and
|
||||
the job commits it. The games it did not reach keep their values until the
|
||||
next run, and since the loops visit their games in an order that changes daily
|
||||
(core/rotation.daily_order) the next cut falls elsewhere.
|
||||
|
||||
A loop that ran out, or used more than three quarters of the budget, says so
|
||||
with a ::warning:: annotation on the run: the sign that the catalogue is
|
||||
outgrowing the job comes before any run is lost.
|
||||
|
||||
JOB_BUDGET_MINUTES unset, empty, zero or not a number means no deadline, so a
|
||||
run on a laptop works as it always did.
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
ENV = "JOB_BUDGET_MINUTES"
|
||||
WARN_SHARE = 0.75
|
||||
|
||||
|
||||
class Deadline:
|
||||
"""A budget of minutes counted from when it was made: from the start of
|
||||
the script, so loading and saving the catalogue count too."""
|
||||
__slots__ = ("start", "seconds")
|
||||
|
||||
def __init__(self, minutes: float, now: Optional[float] = None):
|
||||
self.start = time.monotonic() if now is None else now
|
||||
self.seconds = minutes * 60
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> Optional["Deadline"]:
|
||||
try:
|
||||
minutes = float(os.environ.get(ENV, "").strip())
|
||||
except ValueError:
|
||||
return None
|
||||
if not math.isfinite(minutes) or minutes <= 0:
|
||||
return None
|
||||
return cls(minutes)
|
||||
|
||||
@property
|
||||
def minutes(self) -> float:
|
||||
return self.seconds / 60
|
||||
|
||||
def used(self) -> float:
|
||||
"""The share of the budget spent so far, 0.0 to 1.0 and beyond."""
|
||||
return (time.monotonic() - self.start) / self.seconds
|
||||
|
||||
def expired(self) -> bool:
|
||||
return self.used() >= 1.0
|
||||
|
||||
|
||||
def expired(deadline: Optional[Deadline]) -> bool:
|
||||
return deadline is not None and deadline.expired()
|
||||
|
||||
|
||||
def report(desc: str, done: int, total: int, deadline: Optional[Deadline], log=print) -> None:
|
||||
"""One line on how a loop used the budget. A run that stopped short, or
|
||||
came close, gets a ::warning:: - it is how a growing catalogue shows up in
|
||||
the run's annotations before it costs one."""
|
||||
if deadline is None:
|
||||
return
|
||||
spent = deadline.used() * deadline.minutes
|
||||
if done < total:
|
||||
log(f"::warning::{desc}: stopped after {done} of {total} when the job's "
|
||||
f"{deadline.minutes:.0f}-minute budget ran out; the other {total - done} keep "
|
||||
f"their values until the next run.")
|
||||
elif deadline.used() > WARN_SHARE:
|
||||
log(f"::warning::{desc}: {done} done in {spent:.0f} of the job's {deadline.minutes:.0f} "
|
||||
f"budgeted minutes ({deadline.used():.0%}). The catalogue is outgrowing this "
|
||||
f"schedule: give the job a longer cycle or a larger budget.")
|
||||
else:
|
||||
log(f"{desc}: {done} done in {spent:.0f} of {deadline.minutes:.0f} budgeted minutes.")
|
||||
@@ -7,7 +7,6 @@ import os
|
||||
DATA_DIR = "data"
|
||||
LEGACY_JSONL = "scripts/data.jsonl" # fallback pre-migration
|
||||
TEMP_JSONL = "scripts/temp_info.jsonl"
|
||||
DEAD_LINKS_LOG = "scripts/dead_links.log"
|
||||
REMOVED_JSONL = "scripts/removed_games.jsonl"
|
||||
GAMES_DIR = "games"
|
||||
OVERRIDES_DIR = "data/overrides" # one <appid>.json per human-edited game
|
||||
|
||||
@@ -442,7 +442,7 @@ _SKELETON_TEMPLATE = {
|
||||
def append_note_idempotent(game: dict, marker: str, full_note: str) -> bool:
|
||||
"""Append `full_note` to game['notes'] only if `marker` isn't already present.
|
||||
|
||||
Used by check_dead_links and mark_dead_games to avoid duplicating system
|
||||
Used by mark_dead_games to avoid duplicating system
|
||||
notes when a workflow re-runs over an already-marked game. Returns True
|
||||
when the note was actually changed.
|
||||
"""
|
||||
|
||||
+57
-30
@@ -12,6 +12,7 @@ import time
|
||||
from typing import Optional
|
||||
|
||||
from .anticheat import apply_anti_cheat, from_categories
|
||||
from .budget import Deadline, report
|
||||
from .constants import (
|
||||
STEAM_API_KEY, SKIP_GENRE_TAGS,
|
||||
BATCH_SIZE, BATCH_PAUSE_MIN, BATCH_PAUSE_MAX,
|
||||
@@ -21,6 +22,7 @@ from .data_store import extract_appid, now_iso, is_info_complete, is_empty
|
||||
from .normalize import decode_entities, normalize_release_date
|
||||
from .overrides import OverrideGuard
|
||||
from .peaks import raise_peak
|
||||
from .rotation import daily_order, day_number
|
||||
from .scraper import scrape_store_page
|
||||
from .store_items import paid_dlc_from_items, parse_store_item
|
||||
|
||||
@@ -139,15 +141,27 @@ def apply_details(game: dict, data: dict) -> dict:
|
||||
return game
|
||||
|
||||
|
||||
def reviews_text(total: int, pct: int, label: str) -> str:
|
||||
"""The stored shape of a review summary: "85% (Very Positive)"."""
|
||||
if total <= 0:
|
||||
return "No reviews"
|
||||
return f"{pct}% ({label})" if label else f"{pct}%"
|
||||
|
||||
|
||||
def reviews_from_summary(summary: dict) -> str:
|
||||
"""appreviews' query_summary as stored. The percentage is FLOORED, as
|
||||
Steam floors its own (GetItems answered 85 for Counter-Strike 2 where
|
||||
round() had given 86), so the daily refresh from GetItems
|
||||
(update_reviews.py) and a game's first fetch here agree. In integers:
|
||||
in floating point 29/100*100 is 28.999... and floors to 28."""
|
||||
total = int(summary.get("total_reviews") or 0)
|
||||
pos = int(summary.get("total_positive") or 0)
|
||||
pct = pos * 100 // total if total > 0 else 0
|
||||
return reviews_text(total, pct, summary.get("review_score_desc", ""))
|
||||
|
||||
|
||||
def apply_reviews(game: dict, summary: dict) -> dict:
|
||||
total = summary.get("total_reviews", 0)
|
||||
pos = summary.get("total_positive", 0)
|
||||
if total > 0:
|
||||
pct = round((pos / total) * 100)
|
||||
label = summary.get("review_score_desc", "")
|
||||
game["reviews"] = f"{pct}% ({label})" if label else f"{pct}%"
|
||||
else:
|
||||
game["reviews"] = "No reviews"
|
||||
game["reviews"] = reviews_from_summary(summary)
|
||||
return game
|
||||
|
||||
|
||||
@@ -307,26 +321,44 @@ def fetch_full(game: dict, client=None, fetch_players=True,
|
||||
|
||||
# ──────────── Batch processor ────────────
|
||||
|
||||
def process_batch(games: list[dict], fn, desc="Processing"):
|
||||
"""Run fn on each game with batch pauses."""
|
||||
def process_batch(games: list[dict], fn, desc="Processing", pause: bool = True,
|
||||
deadline: Optional[Deadline] = None) -> int:
|
||||
"""Run fn on each game, BATCH_SIZE at a time. Returns how many it ran on.
|
||||
|
||||
pause: the 15-30 s rest between batches that keeps the store-page and
|
||||
appdetails loops polite. The keyed Web API (player counts) has no
|
||||
need of it: it was half the time top-online took.
|
||||
deadline: a core.budget.Deadline. Once it has passed no further game is
|
||||
started - the one in hand finishes - and the caller saves what was
|
||||
done (core/budget.py). Never pass one where an unvisited game
|
||||
would be lost, as in ingest_new.py, which empties its whole queue.
|
||||
"""
|
||||
total = len(games)
|
||||
done = 0
|
||||
for bs in range(0, total, BATCH_SIZE):
|
||||
if deadline is not None and deadline.expired():
|
||||
break
|
||||
be = min(bs + BATCH_SIZE, total)
|
||||
batch_num = bs // BATCH_SIZE + 1
|
||||
total_batches = (total + BATCH_SIZE - 1) // BATCH_SIZE
|
||||
print(f"\n── Batch {batch_num}/{total_batches} ({be - bs} games) ──")
|
||||
for i in range(bs, be):
|
||||
if deadline is not None and deadline.expired():
|
||||
break
|
||||
name = games[i].get("name") or games[i].get("link", "?")
|
||||
print(f" [{i+1}/{total}] {desc}: {name[:50]}...", end=" ")
|
||||
fn(games[i])
|
||||
done += 1
|
||||
print("✓")
|
||||
if be < total:
|
||||
pause = random.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX)
|
||||
print(f" ⏳ {pause:.0f}s...")
|
||||
time.sleep(pause)
|
||||
if pause and done == be and be < total:
|
||||
wait = random.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX)
|
||||
print(f" ⏳ {wait:.0f}s...")
|
||||
time.sleep(wait)
|
||||
report(desc, done, total, deadline)
|
||||
return done
|
||||
|
||||
|
||||
def update_all_full(games, force=False):
|
||||
def update_all_full(games, force=False, deadline: Optional[Deadline] = None):
|
||||
client = get_client()
|
||||
to_fetch = games if force else [g for g in games if not is_info_complete(g)]
|
||||
print(f"Full update: {len(to_fetch)} to fetch, {len(games)-len(to_fetch)} skipped")
|
||||
@@ -334,22 +366,12 @@ def update_all_full(games, force=False):
|
||||
print("All up to date!")
|
||||
return
|
||||
guard = OverrideGuard.load()
|
||||
process_batch(to_fetch, lambda g: fetch_full(g, client=client, guard=guard), "Fetching")
|
||||
process_batch(daily_order(to_fetch, day_number()),
|
||||
lambda g: fetch_full(g, client=client, guard=guard), "Fetching", deadline=deadline)
|
||||
|
||||
|
||||
def update_reviews_only(games):
|
||||
client = get_client()
|
||||
print(f"Reviews update: {len(games)} games")
|
||||
def fn(g):
|
||||
appid = extract_appid(g.get("link", ""))
|
||||
if not appid: return
|
||||
s = client.fetch_reviews(appid)
|
||||
if s: apply_reviews(g, s)
|
||||
g["last_updated"] = now_iso()
|
||||
process_batch(games, fn, "Reviews")
|
||||
|
||||
|
||||
def update_players_only(games, type_filter: str = "online"):
|
||||
def update_players_only(games, type_filter: str = "online",
|
||||
deadline: Optional[Deadline] = None, day: Optional[int] = None):
|
||||
"""Refresh `current_players` + `peak_today` for the games matching
|
||||
`type_filter`. Default 'online' preserves the historical behaviour
|
||||
used by top_online.py.
|
||||
@@ -360,6 +382,10 @@ def update_players_only(games, type_filter: str = "online"):
|
||||
(single-player, story, etc). Empty type_game is
|
||||
excluded — those are unclassified, not offline.
|
||||
- "all" → every game with a non-empty type_game.
|
||||
|
||||
The keyed Web API is asked without process_batch's pauses, in an order
|
||||
that changes daily, so a run that meets its deadline leaves a different
|
||||
tail unsampled each time.
|
||||
"""
|
||||
if not STEAM_API_KEY:
|
||||
print("No STEAM_API_KEY.")
|
||||
@@ -380,4 +406,5 @@ def update_players_only(games, type_filter: str = "online"):
|
||||
if not appid: return
|
||||
count = client.fetch_player_count(appid, STEAM_API_KEY)
|
||||
if count is not None: apply_players(g, count)
|
||||
process_batch(target, fn, "Players")
|
||||
target = daily_order(target, day_number() if day is None else day)
|
||||
process_batch(target, fn, "Players", pause=False, deadline=deadline)
|
||||
@@ -44,6 +44,12 @@ class HealthResult:
|
||||
def reason(self): return REASON_LABELS.get(self.status, self.status)
|
||||
|
||||
|
||||
def free_per_appdetails(data: dict) -> bool:
|
||||
"""Whether appdetails calls a game free: is_free, or a price that starts at 0."""
|
||||
price = data.get("price_overview") or {}
|
||||
return bool(data.get("is_free", False)) or (bool(price) and price.get("initial", 1) == 0)
|
||||
|
||||
|
||||
def check_game_health(link: str, name: str = "",
|
||||
client: Optional[SteamClient] = None) -> HealthResult:
|
||||
"""
|
||||
@@ -70,11 +76,7 @@ def check_game_health(link: str, name: str = "",
|
||||
# Reject unreleased games – an F2P tracker only lists playable titles.
|
||||
if data.get("release_date", {}).get("coming_soon", False):
|
||||
return HealthResult(COMING_SOON, appid, resolved, link)
|
||||
is_free = data.get("is_free", False)
|
||||
price = data.get("price_overview", {})
|
||||
if price:
|
||||
is_free = is_free or price.get("initial", 1) == 0
|
||||
if not is_free:
|
||||
if not free_per_appdetails(data):
|
||||
return HealthResult(NOT_FREE, appid, resolved, link)
|
||||
# Healthy – return data so caller can reuse it
|
||||
return HealthResult(OK, appid, resolved, link, data=data)
|
||||
|
||||
@@ -97,7 +97,7 @@ def preserve_machine_notes(current: str, human: str) -> str:
|
||||
"""Re-attach machine-appended markers that the human value would drop.
|
||||
|
||||
`notes` is a hybrid field: the pipeline appends its own segments to it —
|
||||
check_dead_links adds "Delisted", mark_dead_games adds "Dead game",
|
||||
the retired dead-link check added "Delisted", mark_dead_games adds "Dead game",
|
||||
fetcher adds "No longer free!". Those appends are one-way. mark_dead_games
|
||||
only calls append_note_idempotent() when `is_dead` is still False, so once
|
||||
a game is marked, a stripped marker is NEVER re-appended. A plain
|
||||
|
||||
@@ -5,7 +5,13 @@ day of a cycle, so a daily run touches 1/cycle of the catalogue.
|
||||
A game's slice is crc32(appid) % cycle, NOT appid % cycle: Steam appids are
|
||||
overwhelmingly multiples of 10, so `% 30` would leave 27 of the 30 monthly
|
||||
slices empty and pile the whole catalogue into the other three.
|
||||
|
||||
The cycle grows with the catalogue (cycle_for), so a daily run stays the same
|
||||
size however many games there are; and a run visits its games in an order
|
||||
that changes daily (daily_order), so one that stops at its time budget
|
||||
(core/budget.py) leaves a different tail each day.
|
||||
"""
|
||||
import math
|
||||
import zlib
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Optional
|
||||
@@ -13,6 +19,26 @@ from typing import Optional
|
||||
from .data_store import extract_appid
|
||||
|
||||
|
||||
def cycle_for(count: int, base: int, daily: int) -> int:
|
||||
"""The length of a rotation over `count` games: `base` days while a daily
|
||||
slice stays within `daily` games, then the smallest MULTIPLE of `base`
|
||||
that keeps it there.
|
||||
|
||||
A multiple, because crc32 % (k * base) leaves crc32 % base unchanged: a
|
||||
game stays on the same day of the base cycle, so when the cycle grows its
|
||||
visits only thin out - never two within `base` days. SteamCharts keeps its
|
||||
one page per game a month, and the store page its weekday.
|
||||
"""
|
||||
if count <= 0 or daily <= 0:
|
||||
return base
|
||||
return base * max(1, math.ceil(count / (base * daily)))
|
||||
|
||||
|
||||
def daily_order(games: list[dict], day: int) -> list[dict]:
|
||||
"""The games in an order that is fixed within a day and changes the next."""
|
||||
return sorted(games, key=lambda g: zlib.crc32(f"{day}:{extract_appid(g.get('link', ''))}".encode()))
|
||||
|
||||
|
||||
def slice_of(appid: str, cycle: int) -> int:
|
||||
"""The day of the cycle on which this game is refreshed. Stable forever:
|
||||
crc32 is fixed, so a game keeps its weekday and its day of the month."""
|
||||
|
||||
@@ -259,12 +259,16 @@ class SteamClient:
|
||||
out[str(item["id"])] = item
|
||||
return out
|
||||
|
||||
def fetch_anticheat_items(self, appids, give_up_after: int = 10) -> tuple[dict[str, dict], set[str]]:
|
||||
"""GetItems asking only for extra_details - each game's anti-cheat
|
||||
disclosure - for a whole catalogue: ({appid: item}, ids not answered).
|
||||
def fetch_items(self, appids, data_request: dict,
|
||||
give_up_after: int = 10) -> tuple[dict[str, dict], set[str]]:
|
||||
"""GetItems for a whole catalogue, a batch at a time: ({appid: item},
|
||||
ids not answered). This is how the daily sweeps read every game in a
|
||||
hundred-odd requests instead of one request per game.
|
||||
|
||||
Unlike fetch_store_items() a failed batch costs only its own ids, which
|
||||
the caller must treat as unknown, never as "no disclosure". After
|
||||
the caller must treat as unknown - never as "no reviews", "no
|
||||
disclosure" or "gone from the store". An id that IS answered for but is
|
||||
missing from `items` is one Steam does not know (success 15). After
|
||||
`give_up_after` failed batches in a row the rest is not asked at all.
|
||||
"""
|
||||
ids = [str(a) for a in appids if str(a).isdigit()]
|
||||
@@ -276,7 +280,7 @@ class SteamClient:
|
||||
if in_a_row >= give_up_after:
|
||||
failed.update(chunk)
|
||||
continue
|
||||
items = self._store_items_batch(chunk, {"include_extra_details": True})
|
||||
items = self._store_items_batch(chunk, data_request)
|
||||
if items is None:
|
||||
failed.update(chunk)
|
||||
in_a_row += 1
|
||||
@@ -286,6 +290,11 @@ class SteamClient:
|
||||
out[str(item["id"])] = item
|
||||
return out, failed
|
||||
|
||||
def fetch_anticheat_items(self, appids, give_up_after: int = 10) -> tuple[dict[str, dict], set[str]]:
|
||||
"""fetch_items() asking only for extra_details - each game's anti-cheat
|
||||
disclosure (core/anticheat.py)."""
|
||||
return self.fetch_items(appids, {"include_extra_details": True}, give_up_after)
|
||||
|
||||
def fetch_tag_names(self) -> dict[int, str]:
|
||||
"""{tagid: English name} from IStoreService/GetTagList, fetched once per
|
||||
client. A failure is not cached, and returns {} (no tag is named)."""
|
||||
@@ -343,7 +352,9 @@ class SteamClient:
|
||||
return body if body.get("success") else None
|
||||
|
||||
def check_store_page(self, appid: str) -> int:
|
||||
"""HEAD request only – lightweight dead link check."""
|
||||
"""HEAD the store page: check_game_health()'s last resort when
|
||||
appdetails answers without data. A delisted app redirects to the store
|
||||
front and ends on 200, so this cannot tell one from a live game."""
|
||||
self._throttle_store()
|
||||
try:
|
||||
resp = self._session.head(
|
||||
|
||||
@@ -16,6 +16,10 @@ Checked against stored HTML scrapes of 40 random games: 37 identical tag lists
|
||||
(the other three had drifted since they were scraped), 40 identical language
|
||||
lists, no unknown tag id. Tag ids are named by IStoreService/GetTagList.
|
||||
|
||||
The daily sweeps read the same items for the whole catalogue, a hundred-odd
|
||||
requests where a request per game took hours: review_summary() for
|
||||
update_reviews.py, screen_item() for purge_unhealthy.py.
|
||||
|
||||
Everything here is pure; steam_client.py makes the requests.
|
||||
"""
|
||||
from typing import Iterable, Optional
|
||||
@@ -120,6 +124,62 @@ def parse_store_item(item: dict, tag_names: dict[int, str]) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def review_summary(item: Optional[dict]) -> Optional[tuple[int, int, str]]:
|
||||
"""(review_count, percent_positive, label) from an item asked for with
|
||||
include_reviews - Steam's own summary, the one its store shows - or None
|
||||
when the item says nothing usable: no item, no summary, a count without a
|
||||
percentage. A count of 0 comes back as (0, 0, ""), and is NOT to be
|
||||
trusted on its own: the summary has said 0 for a game with thousands of
|
||||
reviews (845070, 2026-10-23), and lags behind a new game's first ones.
|
||||
|
||||
percent_positive is floored, as on the store, where round() would round
|
||||
up (85 for Counter-Strike 2, which round() had made 86).
|
||||
"""
|
||||
summary = ((item or {}).get("reviews") or {}).get("summary_filtered")
|
||||
if not isinstance(summary, dict):
|
||||
return None
|
||||
count = _int(summary.get("review_count"))
|
||||
if count < 0:
|
||||
return None
|
||||
if count == 0:
|
||||
return (0, 0, "")
|
||||
pct = _int(summary.get("percent_positive"))
|
||||
if not 0 <= pct <= 100:
|
||||
return None
|
||||
return (count, pct, str(summary.get("review_score_label") or "").strip())
|
||||
|
||||
|
||||
# Why Steam's store says a game may no longer belong in the catalogue
|
||||
# (screen_item), for purge_unhealthy.py to confirm with appdetails.
|
||||
UNKNOWN = "unknown" # Steam does not know the app (success 15)
|
||||
HIDDEN = "hidden" # visible: false
|
||||
UNLISTED = "unlisted" # taken off the store
|
||||
COMING_SOON = "coming_soon" # not released (any more, or yet)
|
||||
NOT_FREE = "not_free" # no is_free flag
|
||||
|
||||
|
||||
def screen_item(item: Optional[dict]) -> Optional[str]:
|
||||
"""Why an item asked for with include_release flags its game, or None
|
||||
for a game that is on the store, released and free. `item` None means
|
||||
Steam answered the batch without it: an app it does not know.
|
||||
|
||||
Only a flag, never a verdict: purge_unhealthy.py removes a game only
|
||||
when appdetails agrees. `type` is not read - a free mod is in the
|
||||
catalogue on purpose (Black Reliquary); that check belongs to ingest.
|
||||
"""
|
||||
if item is None:
|
||||
return UNKNOWN
|
||||
if item.get("visible") is False:
|
||||
return HIDDEN
|
||||
if item.get("unlisted"):
|
||||
return UNLISTED
|
||||
if (item.get("release") or {}).get("is_coming_soon"):
|
||||
return COMING_SOON
|
||||
if not item.get("is_free"):
|
||||
return NOT_FREE
|
||||
return None
|
||||
|
||||
|
||||
def paid_dlc_from_items(items: Iterable[dict]) -> Optional[bool]:
|
||||
"""Whether any of these DLC items costs money. None when there is no item to
|
||||
judge by (every DLC hidden, or the request failed): unknown, not "free".
|
||||
|
||||
+205
-41
@@ -1,48 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Purge unhealthy games."""
|
||||
import sys, os, random, time
|
||||
"""
|
||||
Take out the games that left the store, stopped being free or are not out yet. Daily.
|
||||
|
||||
A game is removed only when two of Steam's answers agree:
|
||||
|
||||
1. the screen - IStoreBrowseService/GetItems for the whole catalogue, a
|
||||
hundred-odd requests (core/store_items.screen_item): Steam does not know
|
||||
the app, it is hidden or unlisted, coming soon, or has no is_free flag;
|
||||
2. the confirmation - appdetails for each flagged game, through the same
|
||||
check_game_health() the ingest uses. An "unavailable" (or 404) answer is
|
||||
asked again RECHECK_AFTER seconds later: appdetails has answered
|
||||
success:false for live games in bursts (14 of them on 2026-09-27).
|
||||
|
||||
This used to ask appdetails about every game, one at a time: 5.3 hours at
|
||||
9,800 games, with one answer enough to remove one.
|
||||
|
||||
The rules:
|
||||
- A batch GetItems did not answer flags nothing, and neither does a network
|
||||
error at the confirmation: unknown is never "gone".
|
||||
- Blocklisted games are left to save_main(), which drops them anyway, and
|
||||
are not asked about.
|
||||
- More than --max-candidates flagged, or more than --max-removals confirmed,
|
||||
stops the run with an error BEFORE anything is written: that is a broken
|
||||
answer from Steam, not a wave of delistings.
|
||||
- Removals are logged to scripts/removed_games.jsonl as they always were
|
||||
(reason and status_code from core/health_checker.py): the Worker's
|
||||
reconcile reads that file.
|
||||
- Games removed in the past that GetItems now sees free, listed and released
|
||||
are listed in a ::notice::, so a removal Steam has since undone can be
|
||||
brought back by queueing its link again.
|
||||
|
||||
Usage (from the repository root):
|
||||
python scripts/purge_unhealthy.py
|
||||
python scripts/purge_unhealthy.py --dry-run
|
||||
python scripts/purge_unhealthy.py --max-removals 300 # a deliberate mass removal
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from typing import Optional
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from core.constants import REMOVED_JSONL, BATCH_SIZE, BATCH_PAUSE_MIN, BATCH_PAUSE_MAX
|
||||
from core.data_store import load_main, save_main, load_jsonl, save_jsonl, dedup_removed, now_iso
|
||||
|
||||
from core.blocklist import blocked_appids
|
||||
from core.budget import Deadline, expired
|
||||
from core.constants import REMOVED_JSONL
|
||||
from core.data_store import (
|
||||
dedup_removed, extract_appid, load_jsonl, load_main, now_iso, save_jsonl, save_main,
|
||||
)
|
||||
from core.health_checker import (
|
||||
COMING_SOON, NOT_FOUND_404, NOT_FOUND_410, NOT_FREE, UNAVAILABLE, check_game_health,
|
||||
)
|
||||
from core.steam_client import get_client
|
||||
from core.health_checker import check_game_health
|
||||
from core.store_items import screen_item
|
||||
|
||||
def main():
|
||||
MAX_CANDIDATES = 500
|
||||
MAX_REMOVALS = 100
|
||||
RECHECK_AFTER = 300 # seconds
|
||||
SCREEN_REQUEST = {"include_release": True}
|
||||
# appdetails answers worth asking twice: the ones a bad moment produces.
|
||||
RECHECKED = frozenset({UNAVAILABLE, NOT_FOUND_404, NOT_FOUND_410})
|
||||
# Removal reasons a game can come back from: what Steam says, not a person.
|
||||
RECOVERABLE = frozenset({UNAVAILABLE, NOT_FOUND_404, NOT_FOUND_410, NOT_FREE, COMING_SOON})
|
||||
|
||||
|
||||
def screen(games: list[dict], items: dict, failed: set, blocked: set) -> list[tuple[dict, str]]:
|
||||
"""The games Steam's store flags, with why. A game without an appid is
|
||||
flagged too: check_game_health() calls its link invalid without a request."""
|
||||
out = []
|
||||
for game in games:
|
||||
appid = extract_appid(game.get("link", ""))
|
||||
if not appid:
|
||||
out.append((game, "invalid"))
|
||||
elif appid not in blocked and appid not in failed:
|
||||
why = screen_item(items.get(appid))
|
||||
if why:
|
||||
out.append((game, why))
|
||||
return out
|
||||
|
||||
|
||||
def confirm(candidates: list[tuple[dict, str]], client, deadline: Optional[Deadline] = None,
|
||||
log=print) -> tuple[list, Counter]:
|
||||
"""Ask appdetails about each flagged game. Returns ([(game, HealthResult)]
|
||||
to remove, tally). An unavailable answer is asked again RECHECK_AFTER
|
||||
seconds after the first ask; past the deadline nothing more is asked and
|
||||
the rest stay."""
|
||||
removals: list = []
|
||||
recheck: list[dict] = []
|
||||
tally: Counter = Counter()
|
||||
first_ask = None
|
||||
|
||||
def judge(game: dict, result) -> None:
|
||||
if result.should_remove:
|
||||
removals.append((game, result))
|
||||
elif result.is_healthy:
|
||||
tally["kept"] += 1
|
||||
log(f" ✓ kept {result.appid} {result.name[:40]}: appdetails says it is healthy")
|
||||
else:
|
||||
tally["network"] += 1
|
||||
|
||||
for game, why in candidates:
|
||||
if expired(deadline):
|
||||
tally["unasked"] += 1
|
||||
continue
|
||||
first_ask = time.monotonic() if first_ask is None else first_ask
|
||||
result = check_game_health(game.get("link", ""), game.get("name", ""), client)
|
||||
if result.status in RECHECKED:
|
||||
recheck.append(game)
|
||||
else:
|
||||
judge(game, result)
|
||||
|
||||
if recheck:
|
||||
wait = RECHECK_AFTER - (time.monotonic() - first_ask)
|
||||
if wait > 0:
|
||||
log(f" {len(recheck)} answered as unavailable; asking again in {wait:.0f} s")
|
||||
time.sleep(wait)
|
||||
for game in recheck:
|
||||
if expired(deadline):
|
||||
tally["unasked"] += 1
|
||||
continue
|
||||
judge(game, check_game_health(game.get("link", ""), game.get("name", ""), client))
|
||||
return removals, tally
|
||||
|
||||
|
||||
def recoverable(removed_rows: list[dict], live: set, blocked: set, client) -> list[str]:
|
||||
"""Earlier removals Steam has since undone: "appid name" for each game
|
||||
removed for a store reason that GetItems now sees free, listed and
|
||||
released, and that is neither back in the catalogue nor blocklisted."""
|
||||
names: dict[str, str] = {}
|
||||
for row in removed_rows:
|
||||
appid = str(row.get("appid") or extract_appid(row.get("link", "")) or "")
|
||||
if appid and row.get("status_code") in RECOVERABLE and appid not in live | blocked:
|
||||
names[appid] = (row.get("name") or "").strip()
|
||||
if not names:
|
||||
return []
|
||||
items, failed = client.fetch_items(list(names), SCREEN_REQUEST)
|
||||
return [f"{a} {names[a]}".strip() for a in names
|
||||
if a not in failed and a in items and screen_item(items[a]) is None]
|
||||
|
||||
|
||||
def main(argv: Optional[list[str]] = None) -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0].strip(),
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--dry-run", action="store_true", help="screen and confirm, write nothing")
|
||||
ap.add_argument("--max-candidates", type=int, default=MAX_CANDIDATES,
|
||||
help=f"stop when the screen flags more games than this (default {MAX_CANDIDATES})")
|
||||
ap.add_argument("--max-removals", type=int, default=MAX_REMOVALS,
|
||||
help=f"stop when more games than this are confirmed (default {MAX_REMOVALS})")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
deadline = Deadline.from_env()
|
||||
games = load_main()
|
||||
if not games: print("Empty."); return
|
||||
print(f"Scanning {len(games)} games...\n")
|
||||
if not games:
|
||||
print("Empty.")
|
||||
return 0
|
||||
client = get_client()
|
||||
remove_set, log = set(), []
|
||||
stats = {"ok": 0, "rm": 0, "net": 0}
|
||||
total = len(games)
|
||||
for bs in range(0, total, BATCH_SIZE):
|
||||
be = min(bs + BATCH_SIZE, total)
|
||||
print(f"── Batch {bs//BATCH_SIZE+1} ──")
|
||||
for i in range(bs, be):
|
||||
g = games[i]
|
||||
name = g.get("name") or g.get("link", "?")
|
||||
r = check_game_health(g.get("link",""), name, client)
|
||||
if r.is_healthy:
|
||||
stats["ok"] += 1
|
||||
elif r.should_remove:
|
||||
print(f" [{i+1}/{total}] ❌ {name[:50]} → {r.reason}")
|
||||
remove_set.add(i)
|
||||
log.append({"link":g.get("link",""),"name":r.name or name,
|
||||
"appid":r.appid or "","reason":r.reason,
|
||||
"status_code":r.status,"removed_at":now_iso()})
|
||||
stats["rm"] += 1
|
||||
else:
|
||||
stats["net"] += 1
|
||||
if be < total:
|
||||
time.sleep(random.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX))
|
||||
if remove_set:
|
||||
cleaned = [g for i,g in enumerate(games) if i not in remove_set]
|
||||
save_main(cleaned)
|
||||
existing = load_jsonl(REMOVED_JSONL)
|
||||
save_jsonl(REMOVED_JSONL, dedup_removed(existing + log))
|
||||
print(f"\nPurged {len(remove_set)}, remaining {len(cleaned)}")
|
||||
else:
|
||||
print("\nAll healthy.")
|
||||
print(f"✅{stats['ok']} ❌{stats['rm']} ⚠{stats['net']}")
|
||||
blocked = blocked_appids()
|
||||
ids = [a for a in (extract_appid(g.get("link", "")) for g in games) if a and a not in blocked]
|
||||
print(f"Screening {len(ids)} games with Steam's store API...")
|
||||
items, failed = client.fetch_items(ids, SCREEN_REQUEST)
|
||||
if ids and len(failed) == len(ids):
|
||||
print("::warning::GetItems answered for no game: nothing screened, nothing removed.")
|
||||
return 0
|
||||
if failed:
|
||||
print(f"::warning::GetItems did not answer for {len(failed)} games; they are screened again tomorrow.")
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
candidates = screen(games, items, failed, blocked)
|
||||
flags = Counter(why for _, why in candidates)
|
||||
print(f"{len(candidates)} flagged: " + (", ".join(f"{k} {v}" for k, v in flags.most_common()) or "none"))
|
||||
if len(candidates) > args.max_candidates:
|
||||
print(f"::error::The screen flagged {len(candidates)} games, more than --max-candidates "
|
||||
f"{args.max_candidates}. That is a broken answer, not a wave of delistings: nothing "
|
||||
f"was asked or written. Run with a higher --max-candidates if it is real.")
|
||||
return 2
|
||||
|
||||
removals, tally = confirm(candidates, client, deadline)
|
||||
for game, result in removals:
|
||||
print(f" ❌ {result.appid} {(result.name or game.get('name', ''))[:50]} → {result.reason}")
|
||||
print(f"Confirmed {len(removals)}; kept {tally['kept']} that appdetails calls healthy, "
|
||||
f"{tally['network']} unanswered, {tally['unasked']} not asked (budget)")
|
||||
if len(removals) > args.max_removals:
|
||||
print(f"::error::{len(removals)} removals, more than --max-removals {args.max_removals}: "
|
||||
f"nothing written. Run with a higher --max-removals if they are real.")
|
||||
return 2
|
||||
|
||||
back = recoverable(load_jsonl(REMOVED_JSONL), set(ids), blocked, client)
|
||||
if back:
|
||||
print(f"::notice::{len(back)} removed game(s) are free and on the store again: "
|
||||
f"{'; '.join(back[:20])}{' …' if len(back) > 20 else ''}. Queue a link in "
|
||||
f"scripts/temp_info.jsonl to bring one back.")
|
||||
|
||||
if args.dry_run:
|
||||
print("Dry run: nothing written.")
|
||||
return 0
|
||||
if not removals:
|
||||
print("All healthy.")
|
||||
return 0
|
||||
gone = {id(game) for game, _ in removals}
|
||||
save_main([g for g in games if id(g) not in gone])
|
||||
log = [{"link": g.get("link", ""), "name": r.name or g.get("name", ""), "appid": r.appid or "",
|
||||
"reason": r.reason, "status_code": r.status, "removed_at": now_iso()}
|
||||
for g, r in removals]
|
||||
save_jsonl(REMOVED_JSONL, dedup_removed(load_jsonl(REMOVED_JSONL) + log))
|
||||
print(f"Purged {len(removals)}, remaining {len(games) - len(removals)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+67
-20
@@ -1,15 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Force re-fetch ALL games. MANUAL ONLY.
|
||||
Backup → migrate → clear fetchable → re-fetch → save.
|
||||
Backup → migrate → re-fetch each game from scratch → save.
|
||||
|
||||
At about six seconds a game the whole catalogue takes far longer than a job
|
||||
may run, so it is done in parts: --part K/N takes the games whose
|
||||
crc32(appid) % N is K-1 (core/rotation.py), and the workflow's runs queue
|
||||
one behind another. Each game is cleared and fetched on a copy that replaces
|
||||
it only once appdetails has answered, so a failed request - or the job
|
||||
budget running out (core/budget.py) - leaves every other game exactly as it
|
||||
was. The oldest last_updated goes first: running a part that was cut short
|
||||
again carries on where it stopped.
|
||||
"""
|
||||
import sys, os, shutil
|
||||
import argparse
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from core.budget import Deadline
|
||||
from core.constants import DATA_DIR, STEAM_API_KEY
|
||||
from core.data_store import load_main, save_main, migrate_record, now_iso
|
||||
from core.data_store import extract_appid, load_main, save_main, migrate_record
|
||||
from core.fetcher import fetch_full, process_batch
|
||||
from core.overrides import OverrideGuard
|
||||
from core.rotation import slice_of
|
||||
from core.steam_client import get_client
|
||||
|
||||
CLEARABLE = {
|
||||
@@ -49,13 +67,51 @@ def clear_fetchable(game):
|
||||
game["metacritic"] = "N/A"
|
||||
|
||||
|
||||
def main():
|
||||
def parse_part(text: str) -> tuple[int, int]:
|
||||
"""'K/N' -> (K, N) with 1 <= K <= N."""
|
||||
m = re.fullmatch(r"\s*(\d+)\s*/\s*(\d+)\s*", text or "")
|
||||
if not m or not 1 <= int(m.group(1)) <= int(m.group(2)):
|
||||
raise argparse.ArgumentTypeError(f"expected K/N with 1 <= K <= N, got {text!r}")
|
||||
return int(m.group(1)), int(m.group(2))
|
||||
|
||||
|
||||
def in_part(game: dict, part: tuple[int, int]) -> bool:
|
||||
appid = extract_appid(game.get("link", ""))
|
||||
return bool(appid) and slice_of(appid, part[1]) == part[0] - 1
|
||||
|
||||
|
||||
def refetch_one(game: dict, client, guard) -> bool:
|
||||
"""Re-fetch one game from scratch, in place, or leave it exactly as it was
|
||||
when appdetails does not answer. Returns whether it was re-fetched."""
|
||||
appid = extract_appid(game.get("link", ""))
|
||||
if not appid:
|
||||
return False
|
||||
status, details = client.fetch_app_details_full(appid)
|
||||
if status != "ok" or not details:
|
||||
return False
|
||||
fresh = copy.deepcopy(game)
|
||||
clear_fetchable(fresh)
|
||||
fetch_full(fresh, client=client, prefetched_details=details, guard=guard)
|
||||
game.clear()
|
||||
game.update(fresh)
|
||||
return True
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0].strip())
|
||||
ap.add_argument("--part", type=parse_part, default=(1, 1),
|
||||
help="K/N: re-fetch only the K-th of N parts of the catalogue (default 1/1)")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
deadline = Deadline.from_env()
|
||||
games = load_main()
|
||||
if not games:
|
||||
print("Empty.")
|
||||
return
|
||||
total = len(games)
|
||||
print(f"Force re-fetch: {total} games\n")
|
||||
targets = [g for g in games if in_part(g, args.part)]
|
||||
# Oldest first: a part cut short carries on where it stopped next time.
|
||||
targets.sort(key=lambda g: g.get("last_updated") or "")
|
||||
print(f"Force re-fetch: part {args.part[0]}/{args.part[1]}, {len(targets)} of {len(games)} games\n")
|
||||
|
||||
backup_dir = DATA_DIR + ".bak"
|
||||
if os.path.isdir(backup_dir):
|
||||
@@ -63,30 +119,21 @@ def main():
|
||||
shutil.copytree(DATA_DIR, backup_dir)
|
||||
print(f"Backup -> {backup_dir}")
|
||||
|
||||
# Migrate + clear
|
||||
for g in games:
|
||||
migrate_record(g)
|
||||
clear_fetchable(g)
|
||||
|
||||
print(f"Cleared fetchable fields. Re-fetching...\n")
|
||||
if not STEAM_API_KEY:
|
||||
print("⚠ No STEAM_API_KEY – players skipped")
|
||||
|
||||
client = get_client()
|
||||
guard = OverrideGuard.load()
|
||||
process_batch(games, lambda g: fetch_full(g, client=client, guard=guard), "Re-fetching")
|
||||
|
||||
stamp = now_iso()
|
||||
for g in games:
|
||||
g["last_updated"] = stamp
|
||||
tally: Counter = Counter()
|
||||
process_batch(targets, lambda g: tally.update([refetch_one(g, client, guard)]),
|
||||
"Re-fetching", deadline=deadline)
|
||||
save_main(games)
|
||||
|
||||
# Stats
|
||||
has = lambda k: sum(1 for g in games if g.get(k) and g[k] not in ("", "N/A", [], False))
|
||||
print(f"\n{'═'*50}")
|
||||
print(f"Done {total} games. Backup: {backup_dir}")
|
||||
print(f" name: {has('name')} | dev: {has('developer')} | pub: {has('publisher')}")
|
||||
print(f" plat: {has('platforms')} | lang: {has('languages')} | tags: {has('tags')}")
|
||||
print(f"Re-fetched {tally[True]} of {len(targets)} ({tally[False]} left as they were: "
|
||||
f"appdetails did not answer). Backup: {backup_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Refresh all_time_peak - the most players each game has had at once - from
|
||||
SteamCharts, a thirtieth of the catalogue a day.
|
||||
SteamCharts, a thirtieth of the catalogue a day (less, once that passes
|
||||
PEAK_DAILY games).
|
||||
|
||||
core/peaks.py explains the field; core/steamcharts.py the source and why it is
|
||||
read so carefully. Each run does two things:
|
||||
@@ -12,7 +13,9 @@ read so carefully. Each run does two things:
|
||||
field for the whole catalogue on the first run.
|
||||
2. For the games whose slice is due (crc32(appid) % 30, see core/rotation.py:
|
||||
every game once a month), read SteamCharts' recorded all-time peak and
|
||||
raise the field to it.
|
||||
raise the field to it. Past PEAK_DAILY games a day the cycle grows to
|
||||
60, 90, ... days (rotation.cycle_for), never shorter: SteamCharts is still
|
||||
asked about a game at most once a month.
|
||||
|
||||
The rules:
|
||||
- all_time_peak only ever goes UP. A failed request, an untracked app or a
|
||||
@@ -23,8 +26,9 @@ The rules:
|
||||
never seen; both are normal and simply skipped until next month.
|
||||
- A 403, a 429 or a Cloudflare challenge stops the SteamCharts part of the
|
||||
run (everything already raised is kept and saved), and so do ten errors in
|
||||
a row. The run still succeeds, with a warning: a refusal is SteamCharts'
|
||||
call to make, not a pipeline failure.
|
||||
a row, and so does the job's time budget (core/budget.py). The run still
|
||||
succeeds, with a warning: a refusal is SteamCharts' call to make, not a
|
||||
pipeline failure.
|
||||
- Everything is written by save_main().
|
||||
|
||||
Usage (from the repository root):
|
||||
@@ -41,12 +45,16 @@ from typing import Optional
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from core.budget import Deadline, expired
|
||||
from core.data_store import extract_appid, load_main, migrate_record, save_main
|
||||
from core.peaks import observed_peak, raise_peak
|
||||
from core.rotation import day_number, due
|
||||
from core.rotation import cycle_for, daily_order, day_number, due
|
||||
from core.steamcharts import SteamChartsClient
|
||||
|
||||
PEAK_CYCLE_DAYS = 30
|
||||
# The most SteamCharts pages a day before the cycle grows (rotation.cycle_for):
|
||||
# about 17 minutes at 2-3 s a page.
|
||||
PEAK_DAILY = 400
|
||||
MAX_CONSECUTIVE_ERRORS = 10
|
||||
|
||||
|
||||
@@ -55,13 +63,19 @@ def raise_from_samples(games: list[dict]) -> int:
|
||||
return sum(1 for g in games if raise_peak(g, observed_peak(g)))
|
||||
|
||||
|
||||
def raise_from_steamcharts(targets: list[dict], client, log=print) -> Counter:
|
||||
def raise_from_steamcharts(targets: list[dict], client, log=print,
|
||||
deadline: Optional[Deadline] = None) -> Counter:
|
||||
"""Step 2. Returns a tally of the outcomes: "raised", "unchanged",
|
||||
"untracked", "error", and "skipped" for games never asked because the run
|
||||
stopped early."""
|
||||
tally: Counter = Counter()
|
||||
errors_in_a_row = 0
|
||||
for i, game in enumerate(targets):
|
||||
if expired(deadline):
|
||||
tally["skipped"] += len(targets) - i
|
||||
log(f"::warning::The job's time budget ran out with {len(targets) - i} SteamCharts "
|
||||
f"pages left; those games are asked on their next turn.")
|
||||
break
|
||||
appid = extract_appid(game.get("link", ""))
|
||||
if not appid:
|
||||
continue
|
||||
@@ -101,6 +115,7 @@ def main(argv: Optional[list[str]] = None) -> None:
|
||||
ap.add_argument("--dry-run", action="store_true", help="fetch and report, write nothing")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
deadline = Deadline.from_env()
|
||||
games = load_main()
|
||||
for g in games:
|
||||
migrate_record(g)
|
||||
@@ -109,13 +124,14 @@ def main(argv: Optional[list[str]] = None) -> None:
|
||||
print(f"{len(games)} games; all_time_peak raised from our own samples for {from_samples}")
|
||||
|
||||
day = day_number() if args.day is None else args.day
|
||||
targets = games if args.all else [g for g in games if due(g, PEAK_CYCLE_DAYS, day)]
|
||||
cycle = cycle_for(len(games), PEAK_CYCLE_DAYS, PEAK_DAILY)
|
||||
targets = games if args.all else [g for g in daily_order(games, day) if due(g, cycle, day)]
|
||||
if args.limit is not None:
|
||||
targets = targets[: args.limit]
|
||||
scope = "every game" if args.all else f"slice {day % PEAK_CYCLE_DAYS}/{PEAK_CYCLE_DAYS}"
|
||||
scope = "every game" if args.all else f"slice {day % cycle}/{cycle}"
|
||||
print(f"SteamCharts: {len(targets)} games ({scope})")
|
||||
|
||||
tally = raise_from_steamcharts(targets, SteamChartsClient()) if targets else Counter()
|
||||
tally = raise_from_steamcharts(targets, SteamChartsClient(), deadline=deadline) if targets else Counter()
|
||||
summary = ", ".join(f"{k} {v}" for k, v in tally.most_common()) or "nothing asked"
|
||||
print(f"\nSteamCharts: {summary}")
|
||||
|
||||
|
||||
@@ -18,8 +18,13 @@ Two groups, each on its own rotation, a slice of the catalogue per daily run:
|
||||
developer, publisher, release_date, platforms,
|
||||
metacritic, drm_notes every 30 days
|
||||
|
||||
That is ~630 store pages and ~150 appdetails calls a day - about 40 minutes
|
||||
at the client's throttle - instead of 8,800 requests in one run.
|
||||
That was ~1,400 store pages and ~330 appdetails calls a day at 9,800 games,
|
||||
about an hour at the client's throttle. The cycles grow with the catalogue
|
||||
(core/rotation.cycle_for): past PAGE_DAILY pages or DETAILS_DAILY appdetails
|
||||
a day, a cycle doubles (triples, ...) instead of the run growing, and a game
|
||||
keeps its weekday and its day of the month. A run that still meets its time
|
||||
budget (core/budget.py) saves what it did; the slice is visited in an order
|
||||
that changes daily, so the cut falls elsewhere next time.
|
||||
|
||||
A game's slice is crc32(appid) % cycle, NOT appid % cycle: Steam appids are
|
||||
overwhelmingly multiples of 10, so `% 30` would leave 27 of the 30 monthly
|
||||
@@ -37,7 +42,10 @@ The rules, because these fields were fill-if-empty until now:
|
||||
gate, a delisted app or a changed page layout leaves the stored value alone.
|
||||
- MANUAL_FIELDS are never written (genre, type_game, anti_cheat, safe, ...),
|
||||
and neither is `notes`: health checks and "no longer free" belong to
|
||||
check_dead_links.py, purge_unhealthy.py and mark_dead_games.py.
|
||||
purge_unhealthy.py and mark_dead_games.py. A game appdetails no longer
|
||||
calls free is only listed, in a ::warning:: - purge removes a game only
|
||||
when Steam's store API flags it too, and this is where the opposite case
|
||||
shows.
|
||||
- has_paid_dlc is read only from a real store page - one with a language
|
||||
table or tags - or, for a gated game, from the prices of its DLC. An
|
||||
age-gate page has no DLC section and would read as False; when neither
|
||||
@@ -60,26 +68,31 @@ Usage (from the repository root):
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import zlib
|
||||
from collections import Counter
|
||||
from typing import Optional
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from core.budget import Deadline
|
||||
from core.data_store import (
|
||||
ELLIPSIS, extract_appid, is_empty, load_main, migrate_record, now_iso, save_main,
|
||||
truncate_description,
|
||||
)
|
||||
from core.fetcher import process_batch, scrape_store_fields
|
||||
from core.health_checker import free_per_appdetails
|
||||
from core.normalize import (
|
||||
clean_list, clean_text, decode_entities, normalize_records, normalize_release_date,
|
||||
)
|
||||
# The rotation (crc32 slices) is shared with refresh_peaks.py.
|
||||
from core.rotation import day_number, due
|
||||
from core.rotation import cycle_for, daily_order, day_number, due
|
||||
from core.steam_client import get_client
|
||||
|
||||
PAGE_CYCLE_DAYS = 7
|
||||
DETAILS_CYCLE_DAYS = 30
|
||||
# The most a day's slice may hold before its cycle grows (rotation.cycle_for):
|
||||
# with the repairs, about an hour and a half at the client's throttle.
|
||||
PAGE_DAILY = 2000
|
||||
DETAILS_DAILY = 500
|
||||
# Games with empty store fields taken a day on top of the rotation. 25 on
|
||||
# 2026-10-10; the cap is for the day something breaks for every game at once,
|
||||
# so a two-hour job cannot turn into a whole-catalogue crawl.
|
||||
@@ -205,8 +218,10 @@ def apply_app_details(game: dict, data: dict) -> list[str]:
|
||||
return changed
|
||||
|
||||
|
||||
def refresh_game(game: dict, client, page: bool, details: bool) -> list[str]:
|
||||
"""Refresh one game in place. Returns the fields that changed."""
|
||||
def refresh_game(game: dict, client, page: bool, details: bool,
|
||||
not_free: Optional[list] = None) -> list[str]:
|
||||
"""Refresh one game in place. Returns the fields that changed. A game
|
||||
whose appdetails no longer call it free is added to `not_free`."""
|
||||
appid = extract_appid(game.get("link", ""))
|
||||
if not appid:
|
||||
return []
|
||||
@@ -216,6 +231,8 @@ def refresh_game(game: dict, client, page: bool, details: bool) -> list[str]:
|
||||
data = client.fetch_app_details(appid)
|
||||
if data:
|
||||
changed += apply_app_details(game, data)
|
||||
if not_free is not None and not free_per_appdetails(data):
|
||||
not_free.append(f"{appid} {game.get('name', '')[:40]}")
|
||||
if page:
|
||||
# The appdetails just fetched spare the DLC check a second request.
|
||||
scraped = scrape_store_fields(client, appid, details=data)
|
||||
@@ -233,29 +250,31 @@ def missing_store_fields(game: dict) -> bool:
|
||||
|
||||
|
||||
def build_plan(games: list[dict], day: int, want_page: bool = True,
|
||||
want_details: bool = True, missing_only: bool = False) -> list[tuple]:
|
||||
want_details: bool = True, missing_only: bool = False,
|
||||
page_cycle: int = PAGE_CYCLE_DAYS,
|
||||
details_cycle: int = DETAILS_CYCLE_DAYS) -> list[tuple]:
|
||||
"""[(game, page, details)] for one day: the games with empty store fields
|
||||
first (page group, every day), then the day's rotation slices.
|
||||
|
||||
The repairs are capped at MISSING_DAILY_CAP and ordered by a hash that
|
||||
changes daily, so a residue Steam never fills cannot hold the same places
|
||||
forever. missing_only keeps just the repairs, store page only.
|
||||
Both parts come in an order that changes daily (rotation.daily_order):
|
||||
the repairs are capped at MISSING_DAILY_CAP, so a residue Steam never
|
||||
fills cannot hold the same places forever, and a run that meets its time
|
||||
budget leaves a different part of the slice for the next one.
|
||||
missing_only keeps just the repairs, store page only.
|
||||
"""
|
||||
repairs: list[dict] = []
|
||||
if want_page:
|
||||
repairs = [g for g in games if missing_store_fields(g)]
|
||||
repairs.sort(key=lambda g: zlib.crc32(f"{day}:{extract_appid(g.get('link', ''))}".encode()))
|
||||
repairs = repairs[:MISSING_DAILY_CAP]
|
||||
repairs = daily_order([g for g in games if missing_store_fields(g)], day)[:MISSING_DAILY_CAP]
|
||||
if missing_only:
|
||||
return [(g, True, False) for g in repairs]
|
||||
|
||||
plan = [(g, True, want_details and due(g, DETAILS_CYCLE_DAYS, day)) for g in repairs]
|
||||
plan = [(g, True, want_details and due(g, details_cycle, day)) for g in repairs]
|
||||
taken = {id(g) for g in repairs}
|
||||
for g in games:
|
||||
for g in daily_order(games, day):
|
||||
if id(g) in taken:
|
||||
continue
|
||||
page = want_page and due(g, PAGE_CYCLE_DAYS, day)
|
||||
details = want_details and due(g, DETAILS_CYCLE_DAYS, day)
|
||||
page = want_page and due(g, page_cycle, day)
|
||||
details = want_details and due(g, details_cycle, day)
|
||||
if page or details:
|
||||
plan.append((g, page, details))
|
||||
return plan
|
||||
@@ -273,6 +292,7 @@ def main(argv: Optional[list[str]] = None) -> None:
|
||||
ap.add_argument("--dry-run", action="store_true", help="fetch and report, write nothing")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
deadline = Deadline.from_env()
|
||||
day = day_number() if args.day is None else args.day
|
||||
games = load_main()
|
||||
for g in games:
|
||||
@@ -284,7 +304,10 @@ def main(argv: Optional[list[str]] = None) -> None:
|
||||
|
||||
want_page = args.only in (None, "page")
|
||||
want_details = args.only in (None, "details")
|
||||
plan = build_plan(games, day, want_page, want_details, missing_only=args.missing_only)
|
||||
page_cycle = cycle_for(len(games), PAGE_CYCLE_DAYS, PAGE_DAILY)
|
||||
details_cycle = cycle_for(len(games), DETAILS_CYCLE_DAYS, DETAILS_DAILY)
|
||||
plan = build_plan(games, day, want_page, want_details, missing_only=args.missing_only,
|
||||
page_cycle=page_cycle, details_cycle=details_cycle)
|
||||
if args.limit is not None:
|
||||
plan = plan[: args.limit]
|
||||
|
||||
@@ -292,28 +315,34 @@ def main(argv: Optional[list[str]] = None) -> None:
|
||||
n_details = sum(1 for _, _, d in plan if d)
|
||||
n_missing = sum(1 for g, p, _ in plan if p and missing_store_fields(g))
|
||||
print(f"{len(games)} games; day {day}: store page for {n_page} "
|
||||
f"(slice {day % PAGE_CYCLE_DAYS}/{PAGE_CYCLE_DAYS}, {n_missing} with empty store fields), "
|
||||
f"appdetails for {n_details} (slice {day % DETAILS_CYCLE_DAYS}/{DETAILS_CYCLE_DAYS})")
|
||||
f"(slice {day % page_cycle}/{page_cycle}, {n_missing} with empty store fields), "
|
||||
f"appdetails for {n_details} (slice {day % details_cycle}/{details_cycle})")
|
||||
if not plan:
|
||||
return
|
||||
|
||||
client = get_client()
|
||||
flags = {id(g): (page, details) for g, page, details in plan}
|
||||
fields: Counter = Counter()
|
||||
not_free: list[str] = []
|
||||
touched = 0
|
||||
|
||||
def refresh(game: dict) -> None:
|
||||
nonlocal touched
|
||||
page, details = flags[id(game)]
|
||||
changed = refresh_game(game, client, page=page, details=details)
|
||||
changed = refresh_game(game, client, page=page, details=details, not_free=not_free)
|
||||
if changed:
|
||||
touched += 1
|
||||
fields.update(changed)
|
||||
|
||||
process_batch([g for g, _, _ in plan], refresh, "Refreshing")
|
||||
done = process_batch([g for g, _, _ in plan], refresh, "Refreshing", deadline=deadline)
|
||||
|
||||
summary = ", ".join(f"{f} {n}" for f, n in fields.most_common()) or "nothing"
|
||||
print(f"\n{touched} of {len(plan)} games changed: {summary}")
|
||||
print(f"\n{touched} of {done} games changed: {summary}")
|
||||
if not_free:
|
||||
print(f"::warning::appdetails no longer calls {len(not_free)} game(s) free: "
|
||||
f"{'; '.join(not_free[:20])}{' …' if len(not_free) > 20 else ''}. The daily purge "
|
||||
f"removes one only when Steam's store API says so too; if it does not, remove it "
|
||||
f"in /admin.")
|
||||
if args.dry_run:
|
||||
print("Dry run: nothing written.")
|
||||
return
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
"""
|
||||
A job's time budget and the rotations that keep a daily run the same size
|
||||
however large the catalogue grows. These pin that a budget is opt-in, that a
|
||||
loop past its deadline starts no further game, that only the store loops
|
||||
pause, and that a growing cycle never brings a game's visits closer together.
|
||||
|
||||
Run from the repository root: python -m pytest scripts/tests
|
||||
"""
|
||||
import zlib
|
||||
|
||||
import pytest
|
||||
|
||||
from core import budget, fetcher, rotation
|
||||
from core.budget import Deadline
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self, now=1000.0):
|
||||
self.now = now
|
||||
|
||||
def __call__(self):
|
||||
return self.now
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clock(monkeypatch):
|
||||
c = Clock()
|
||||
monkeypatch.setattr(budget.time, "monotonic", c)
|
||||
return c
|
||||
|
||||
|
||||
def _games(n: int) -> list[dict]:
|
||||
return [{"link": f"https://store.steampowered.com/app/{(i + 1) * 10}/", "name": f"G{i}"} for i in range(n)]
|
||||
|
||||
|
||||
# ──────────── Deadline ────────────
|
||||
|
||||
@pytest.mark.parametrize("raw", [None, "", "0", "-5", "soon", "nan", "inf"])
|
||||
def test_no_budget_means_no_deadline(monkeypatch, raw):
|
||||
if raw is None:
|
||||
monkeypatch.delenv(budget.ENV, raising=False)
|
||||
else:
|
||||
monkeypatch.setenv(budget.ENV, raw)
|
||||
assert Deadline.from_env() is None
|
||||
assert budget.expired(None) is False
|
||||
|
||||
|
||||
def test_a_budget_counts_from_when_it_was_made(monkeypatch, clock):
|
||||
monkeypatch.setenv(budget.ENV, " 30 ")
|
||||
deadline = Deadline.from_env()
|
||||
assert deadline.minutes == 30
|
||||
clock.now += 29 * 60
|
||||
assert not budget.expired(deadline)
|
||||
clock.now += 60
|
||||
assert budget.expired(deadline) and deadline.used() == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_the_report_warns_when_a_loop_ran_out_or_came_close(clock):
|
||||
lines: list[str] = []
|
||||
deadline = Deadline(10)
|
||||
clock.now += 60
|
||||
budget.report("Players", 5, 5, deadline, log=lines.append)
|
||||
clock.now += 7 * 60
|
||||
budget.report("Players", 5, 5, deadline, log=lines.append)
|
||||
budget.report("Players", 3, 5, deadline, log=lines.append)
|
||||
budget.report("Players", 5, 5, None, log=lines.append)
|
||||
assert not lines[0].startswith("::warning::") and "1 of 10" in lines[0]
|
||||
assert lines[1].startswith("::warning::") and "outgrowing" in lines[1]
|
||||
assert lines[2].startswith("::warning::") and "stopped after 3 of 5" in lines[2]
|
||||
assert len(lines) == 3
|
||||
|
||||
|
||||
# ──────────── process_batch ────────────
|
||||
|
||||
def test_past_the_deadline_no_further_game_is_started(monkeypatch, clock):
|
||||
monkeypatch.setattr(fetcher.time, "sleep", lambda s: None)
|
||||
deadline = Deadline(1)
|
||||
seen: list[str] = []
|
||||
|
||||
def fn(game):
|
||||
seen.append(game["name"])
|
||||
clock.now += 25 # the third game crosses the minute
|
||||
|
||||
assert fetcher.process_batch(_games(10), fn, "Test", deadline=deadline) == 3
|
||||
assert seen == ["G0", "G1", "G2"]
|
||||
|
||||
|
||||
def test_only_the_store_loops_pause_between_batches(monkeypatch):
|
||||
monkeypatch.setattr(fetcher, "BATCH_SIZE", 2)
|
||||
pauses: list[float] = []
|
||||
monkeypatch.setattr(fetcher.time, "sleep", pauses.append)
|
||||
assert fetcher.process_batch(_games(5), lambda g: None, "Store") == 5
|
||||
assert len(pauses) == 2 # between the three batches, not after the last
|
||||
pauses.clear()
|
||||
assert fetcher.process_batch(_games(5), lambda g: None, "Players", pause=False) == 5
|
||||
assert pauses == []
|
||||
|
||||
|
||||
def test_players_are_asked_without_pauses_in_a_daily_order(monkeypatch):
|
||||
monkeypatch.setattr(fetcher, "STEAM_API_KEY", "key")
|
||||
monkeypatch.setattr(fetcher.time, "sleep", lambda s: pytest.fail("the Web API loop slept"))
|
||||
asked: list[str] = []
|
||||
|
||||
class Client:
|
||||
def fetch_player_count(self, appid, key):
|
||||
asked.append(appid)
|
||||
return 7
|
||||
|
||||
monkeypatch.setattr(fetcher, "get_client", lambda: Client())
|
||||
games = [dict(g, type_game="online") for g in _games(120)]
|
||||
fetcher.update_players_only(games, day=100)
|
||||
first = list(asked)
|
||||
asked.clear()
|
||||
fetcher.update_players_only(games, day=101)
|
||||
assert sorted(first) == sorted(asked) and first != asked
|
||||
assert all(g["current_players"] == "7" for g in games)
|
||||
|
||||
|
||||
def test_the_full_update_stops_at_its_deadline(monkeypatch, clock):
|
||||
monkeypatch.setattr(fetcher.time, "sleep", lambda s: None)
|
||||
monkeypatch.setattr(fetcher, "get_client", lambda: object())
|
||||
monkeypatch.setattr(fetcher.OverrideGuard, "load", classmethod(lambda cls: None))
|
||||
fetched: list[str] = []
|
||||
|
||||
def fake_fetch(game, client=None, guard=None):
|
||||
fetched.append(game["name"])
|
||||
clock.now += 61
|
||||
|
||||
monkeypatch.setattr(fetcher, "fetch_full", fake_fetch)
|
||||
fetcher.update_all_full(_games(4), deadline=Deadline(2))
|
||||
assert len(fetched) == 2
|
||||
|
||||
|
||||
# ──────────── Rotation ────────────
|
||||
|
||||
@pytest.mark.parametrize("base, daily", [(7, 2000), (30, 500), (30, 400)])
|
||||
def test_a_cycle_is_a_multiple_of_its_base_that_never_shrinks(base, daily):
|
||||
last = base
|
||||
for count in range(0, 100_001, 997):
|
||||
cycle = rotation.cycle_for(count, base, daily)
|
||||
assert cycle % base == 0 and cycle >= last
|
||||
assert count / cycle <= daily or cycle == base and count <= base * daily
|
||||
last = cycle
|
||||
|
||||
|
||||
def test_the_cycle_stays_at_its_base_until_a_day_would_overflow():
|
||||
assert rotation.cycle_for(9847, 7, 2000) == 7
|
||||
assert rotation.cycle_for(14000, 7, 2000) == 7
|
||||
assert rotation.cycle_for(14001, 7, 2000) == 14
|
||||
assert rotation.cycle_for(12000, 30, 400) == 30
|
||||
assert rotation.cycle_for(12001, 30, 400) == 60
|
||||
assert rotation.cycle_for(5, 7, 0) == 7
|
||||
|
||||
|
||||
def test_a_growing_cycle_keeps_every_visit_at_least_a_base_apart():
|
||||
base = 30
|
||||
for appid in ("730", "570", "2507950", "4219930"):
|
||||
crc = zlib.crc32(appid.encode())
|
||||
visits = []
|
||||
for day in range(20000, 20400):
|
||||
cycle = 30 if day < 20150 else 60 if day < 20300 else 90
|
||||
if rotation.due({"link": f"https://store.steampowered.com/app/{appid}/"}, cycle, day):
|
||||
visits.append(day)
|
||||
assert all(day % base == crc % base for day in visits)
|
||||
assert all(b - a >= base for a, b in zip(visits, visits[1:]))
|
||||
|
||||
|
||||
def test_the_daily_order_holds_within_a_day_and_changes_the_next():
|
||||
games = _games(50)
|
||||
assert rotation.daily_order(games, 7) == rotation.daily_order(list(reversed(games)), 7)
|
||||
assert rotation.daily_order(games, 7) != rotation.daily_order(games, 8)
|
||||
assert sorted(g["name"] for g in rotation.daily_order(games, 8)) == sorted(g["name"] for g in games)
|
||||
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
check_game_health() is the confirmation every removal goes through: purge
|
||||
asks it about the games Steam's store API flags, and ingest about every queued
|
||||
link. These pin how appdetails' answers map to a verdict.
|
||||
|
||||
Steam is never called: the client is a fake. Run from the repository root:
|
||||
python -m pytest scripts/tests
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from core import health_checker as hc
|
||||
|
||||
LINK = "https://store.steampowered.com/app/730/"
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, status="ok", data=None, head=200):
|
||||
self.status, self.data, self.head = status, data, head
|
||||
|
||||
def fetch_app_details_full(self, appid):
|
||||
return (self.status, self.data)
|
||||
|
||||
def check_store_page(self, appid):
|
||||
return self.head
|
||||
|
||||
|
||||
@pytest.mark.parametrize("data, expected", [
|
||||
({"is_free": True}, True),
|
||||
({"is_free": False, "price_overview": {"initial": 0}}, True),
|
||||
({"is_free": False, "price_overview": {"initial": 1499}}, False),
|
||||
({"is_free": False, "price_overview": None}, False),
|
||||
({}, False),
|
||||
])
|
||||
def test_what_appdetails_calls_free(data, expected):
|
||||
assert hc.free_per_appdetails(data) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client, status", [
|
||||
(FakeClient(data={"name": "CS2", "is_free": True}), hc.OK),
|
||||
(FakeClient(data={"name": "CS2", "is_free": False, "price_overview": {"initial": 999}}), hc.NOT_FREE),
|
||||
(FakeClient(data={"name": "CS2", "is_free": True, "release_date": {"coming_soon": True}}), hc.COMING_SOON),
|
||||
(FakeClient(status="unavailable"), hc.UNAVAILABLE),
|
||||
(FakeClient(status="not_found"), hc.NOT_FOUND_404),
|
||||
(FakeClient(status="network_error"), hc.NETWORK_ERROR),
|
||||
(FakeClient(data={}, head=404), hc.NOT_FOUND_404),
|
||||
(FakeClient(data={}, head=410), hc.NOT_FOUND_410),
|
||||
(FakeClient(data={}, head=200), hc.OK),
|
||||
(FakeClient(data={}, head=-1), hc.NETWORK_ERROR),
|
||||
])
|
||||
def test_appdetails_answers_map_to_a_verdict(client, status):
|
||||
result = hc.check_game_health(LINK, "Counter-Strike 2", client)
|
||||
assert result.status == status and result.appid == "730"
|
||||
assert result.should_remove is (status in hc.REMOVABLE)
|
||||
|
||||
|
||||
def test_a_link_without_an_appid_is_invalid_without_a_request():
|
||||
result = hc.check_game_health("https://example.com/", "x", object())
|
||||
assert result.status == hc.INVALID_FORMAT and result.should_remove
|
||||
@@ -0,0 +1,76 @@
|
||||
"""
|
||||
The leaderboards sample every online (top_online.py) or offline
|
||||
(top_offline.py) game's player count through the keyed Web API, and
|
||||
update_data.py refills incomplete records. These run each script end to end
|
||||
against a fake client: they write their page and save through save_main(),
|
||||
and the Web API loop never pauses.
|
||||
|
||||
Run from the repository root: python -m pytest scripts/tests
|
||||
"""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import top_offline
|
||||
import top_online
|
||||
import update_data
|
||||
from core import budget, data_store, fetcher
|
||||
|
||||
|
||||
def _rec(appid: str, type_game: str, players: str = "10") -> dict:
|
||||
return {"link": f"https://store.steampowered.com/app/{appid}/", "name": f"Game {appid}",
|
||||
"type_game": type_game, "status": "active", "current_players": players,
|
||||
"peak_today": players, "reviews": "90% (Very Positive)", "genre": "Action",
|
||||
"anti_cheat": "-", "notes": "Not reviewed yet", "last_updated": "2026-01-01T00:00:00Z"}
|
||||
|
||||
|
||||
class Players:
|
||||
def __init__(self, counts):
|
||||
self.counts = counts
|
||||
|
||||
def fetch_player_count(self, appid, key):
|
||||
return self.counts.get(appid)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
os.makedirs("data")
|
||||
monkeypatch.delenv(budget.ENV, raising=False)
|
||||
monkeypatch.setattr(fetcher, "STEAM_API_KEY", "key")
|
||||
monkeypatch.setattr(fetcher.time, "sleep", lambda s: pytest.fail("the Web API loop slept"))
|
||||
data_store.save_main([_rec("10", "online"), _rec("20", "online"), _rec("30", "offline")],
|
||||
apply_human_overrides=False)
|
||||
return monkeypatch
|
||||
|
||||
|
||||
def _after() -> dict:
|
||||
return {data_store.extract_appid(g["link"]): g for g in data_store.load_main()}
|
||||
|
||||
|
||||
def test_top_online_samples_the_online_games_and_writes_its_page(repo):
|
||||
repo.setattr(top_online, "STEAM_API_KEY", "key")
|
||||
repo.setattr(fetcher, "get_client", lambda: Players({"10": 50000, "20": 3}))
|
||||
top_online.main()
|
||||
page = open(os.path.join("games", "top-online.md"), encoding="utf-8").read()
|
||||
assert page.index("Game 10") < page.index("Game 20") and "Game 30" not in page
|
||||
after = _after()
|
||||
assert (after["10"]["current_players"], after["30"]["current_players"]) == ("50,000", "10")
|
||||
|
||||
|
||||
def test_top_offline_samples_the_offline_games_and_writes_its_page(repo):
|
||||
repo.setattr(top_offline, "STEAM_API_KEY", "key")
|
||||
repo.setattr(fetcher, "get_client", lambda: Players({"30": 1200}))
|
||||
top_offline.main()
|
||||
page = open(os.path.join("games", "top-offline.md"), encoding="utf-8").read()
|
||||
assert "Game 30" in page and "Game 10" not in page
|
||||
assert _after()["30"]["current_players"] == "1,200"
|
||||
|
||||
|
||||
def test_update_data_refills_only_incomplete_records(repo):
|
||||
refilled: list[str] = []
|
||||
repo.setattr(fetcher, "get_client", lambda: object())
|
||||
repo.setattr(fetcher.OverrideGuard, "load", classmethod(lambda cls: None))
|
||||
repo.setattr(fetcher, "fetch_full", lambda g, client=None, guard=None: refilled.append(g["link"]))
|
||||
update_data.main()
|
||||
assert len(refilled) == 3 # the fixture's records lack most store fields
|
||||
@@ -0,0 +1,214 @@
|
||||
"""
|
||||
purge_unhealthy.py removes a game only when two of Steam's answers agree: the
|
||||
store API's screen of the whole catalogue, and appdetails for each game it
|
||||
flags. These pin the rules that make a daily, unattended removal safe: what
|
||||
nobody answered is never "gone", appdetails' unavailable is asked twice,
|
||||
blocklisted games are not asked about, a broken answer stops the run before
|
||||
anything is written, and removals are logged as the Worker's reconcile reads
|
||||
them.
|
||||
|
||||
Steam is never called: the client and appdetails are fakes. Run from the
|
||||
repository root: python -m pytest scripts/tests
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import purge_unhealthy as pu
|
||||
from core import budget, data_store
|
||||
from core.health_checker import (
|
||||
NETWORK_ERROR, NOT_FREE, OK, UNAVAILABLE, HealthResult,
|
||||
)
|
||||
|
||||
OLD = "2026-01-01T00:00:00Z"
|
||||
FREE = {"visible": True, "is_free": True, "release": {"steam_release_date": 1}}
|
||||
|
||||
|
||||
def _rec(appid: str, name: str = "") -> dict:
|
||||
return {"link": f"https://store.steampowered.com/app/{appid}/", "name": name or f"Game {appid}",
|
||||
"type_game": "online", "notes": "Not reviewed yet", "last_updated": OLD}
|
||||
|
||||
|
||||
def _item(appid: str, **fields) -> dict:
|
||||
return dict(FREE, id=int(appid), success=1, **fields)
|
||||
|
||||
|
||||
class FakeClient:
|
||||
"""GetItems items by appid (absent: unknown to Steam) and failed appids."""
|
||||
|
||||
def __init__(self, items=None, failed=()):
|
||||
self.items, self.failed = items or {}, set(failed)
|
||||
self.asked: list[list[str]] = []
|
||||
|
||||
def fetch_items(self, appids, data_request, give_up_after=10):
|
||||
assert data_request == {"include_release": True}
|
||||
self.asked.append(list(appids))
|
||||
return ({a: self.items[a] for a in appids if a in self.items and a not in self.failed},
|
||||
{a for a in appids if a in self.failed})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
os.makedirs("data")
|
||||
os.makedirs("scripts")
|
||||
monkeypatch.delenv(budget.ENV, raising=False)
|
||||
sleeps: list[float] = []
|
||||
monkeypatch.setattr(pu.time, "sleep", sleeps.append)
|
||||
answers: dict[str, list[str]] = {}
|
||||
health_calls: list[str] = []
|
||||
|
||||
def fake_health(link, name="", client=None):
|
||||
appid = data_store.extract_appid(link)
|
||||
health_calls.append(appid)
|
||||
status = answers[appid].pop(0)
|
||||
return HealthResult(status, appid, name, link)
|
||||
|
||||
monkeypatch.setattr(pu, "check_game_health", fake_health)
|
||||
monkeypatch.setattr(pu, "blocked_appids", lambda: set())
|
||||
|
||||
class Repo:
|
||||
def run(self, records=None, client=None, health=None, argv=()):
|
||||
if records is not None:
|
||||
data_store.save_main(records, apply_human_overrides=False)
|
||||
answers.clear()
|
||||
answers.update({a: list(v) for a, v in (health or {}).items()})
|
||||
monkeypatch.setattr(pu, "get_client", lambda: client)
|
||||
code = pu.main(list(argv))
|
||||
return code, {data_store.extract_appid(g["link"]) for g in data_store.load_main()}
|
||||
|
||||
r = Repo()
|
||||
r.sleeps, r.health_calls = sleeps, health_calls
|
||||
r.block = lambda ids: monkeypatch.setattr(pu, "blocked_appids", lambda: set(ids))
|
||||
return r
|
||||
|
||||
|
||||
def _removed() -> list[dict]:
|
||||
path = os.path.join("scripts", "removed_games.jsonl")
|
||||
return [json.loads(line) for line in open(path, encoding="utf-8")] if os.path.exists(path) else []
|
||||
|
||||
|
||||
def _bytes() -> dict:
|
||||
files = [os.path.join("data", n) for n in sorted(os.listdir("data")) if n.endswith((".jsonl", ".json"))]
|
||||
files += [os.path.join("scripts", n) for n in sorted(os.listdir("scripts"))]
|
||||
return {f: open(f, "rb").read() for f in files}
|
||||
|
||||
|
||||
def test_an_empty_catalogue_asks_nothing(repo):
|
||||
client = FakeClient()
|
||||
assert repo.run(None, client) == (0, set())
|
||||
assert client.asked == []
|
||||
|
||||
|
||||
def test_a_healthy_catalogue_asks_appdetails_nothing(repo):
|
||||
client = FakeClient(items={"730": _item("730"), "570": _item("570")})
|
||||
code, live = repo.run([_rec("730"), _rec("570")], client)
|
||||
assert (code, live) == (0, {"730", "570"})
|
||||
assert repo.health_calls == [] and _removed() == []
|
||||
|
||||
|
||||
def test_a_game_both_answers_flag_is_removed_and_logged(repo):
|
||||
client = FakeClient(items={"730": _item("730"), "4991340": _item("4991340", unlisted=True, is_free=False)})
|
||||
code, live = repo.run([_rec("730"), _rec("4991340", "Might & Poison Mushroom")], client,
|
||||
health={"4991340": [NOT_FREE]})
|
||||
assert (code, live) == (0, {"730"})
|
||||
[row] = _removed()
|
||||
assert (row["appid"], row["status_code"], row["reason"]) == ("4991340", "not_free", "No longer free-to-play")
|
||||
assert row["name"] == "Might & Poison Mushroom" and row["removed_at"]
|
||||
|
||||
|
||||
def test_a_flag_appdetails_contradicts_is_kept(repo):
|
||||
client = FakeClient(items={"2878670": _item("2878670", is_free=False)})
|
||||
code, live = repo.run([_rec("2878670", "Spoop Troop")], client, health={"2878670": [OK]})
|
||||
assert (code, live) == (0, {"2878670"}) and _removed() == []
|
||||
|
||||
|
||||
def test_unavailable_is_asked_again_before_a_game_goes(repo):
|
||||
client = FakeClient(items={}) # answered, and unknown to Steam
|
||||
code, live = repo.run([_rec("10"), _rec("20")], client,
|
||||
health={"10": [UNAVAILABLE, UNAVAILABLE], "20": [UNAVAILABLE, OK]})
|
||||
assert code == 0 and live == {"20"}
|
||||
assert repo.health_calls == ["10", "20", "10", "20"]
|
||||
assert len(repo.sleeps) == 1 and 0 < repo.sleeps[0] <= pu.RECHECK_AFTER
|
||||
assert [r["status_code"] for r in _removed()] == ["unavailable"]
|
||||
|
||||
|
||||
def test_what_nobody_answered_is_never_gone(repo):
|
||||
client = FakeClient(items={"730": _item("730")}, failed={"570"})
|
||||
code, live = repo.run([_rec("730"), _rec("570"), _rec("440")], client,
|
||||
health={"440": [NETWORK_ERROR]})
|
||||
assert (code, live) == (0, {"730", "570", "440"})
|
||||
assert repo.health_calls == ["440"] # 570's batch failed: not even asked
|
||||
|
||||
|
||||
def test_no_answer_at_all_writes_nothing(repo):
|
||||
repo.run([_rec("730")], FakeClient(items={"730": _item("730")}))
|
||||
before = _bytes()
|
||||
code, live = repo.run(None, FakeClient(failed={"730"}))
|
||||
assert (code, live) == (0, {"730"}) and _bytes() == before
|
||||
|
||||
|
||||
def test_a_blocklisted_game_is_not_asked_about(repo):
|
||||
repo.block({"570"})
|
||||
client = FakeClient(items={"730": _item("730")})
|
||||
repo.run([_rec("730"), _rec("570")], client)
|
||||
assert "570" not in client.asked[0] and repo.health_calls == []
|
||||
|
||||
|
||||
def test_too_many_flags_stop_the_run_before_any_request(repo):
|
||||
repo.run([_rec("730"), _rec("570"), _rec("440")], FakeClient(items={a: _item(a) for a in ("730", "570", "440")}))
|
||||
before = _bytes()
|
||||
code, _ = repo.run(None, FakeClient(items={}), argv=["--max-candidates", "2"])
|
||||
assert code == 2 and repo.health_calls == [] and _bytes() == before
|
||||
|
||||
|
||||
def test_too_many_removals_stop_the_run_before_anything_is_written(repo):
|
||||
repo.run([_rec("730"), _rec("570")], FakeClient(items={"730": _item("730"), "570": _item("570")}))
|
||||
before = _bytes()
|
||||
client = FakeClient(items={"730": _item("730", is_free=False), "570": _item("570", is_free=False)})
|
||||
code, live = repo.run(None, client, health={"730": [NOT_FREE], "570": [NOT_FREE]},
|
||||
argv=["--max-removals", "1"])
|
||||
assert code == 2 and live == {"730", "570"} and _bytes() == before
|
||||
|
||||
|
||||
def test_a_dry_run_writes_nothing(repo):
|
||||
repo.run([_rec("730")], FakeClient(items={"730": _item("730")}))
|
||||
before = _bytes()
|
||||
code, _ = repo.run(None, FakeClient(items={"730": _item("730", unlisted=True)}),
|
||||
health={"730": [NOT_FREE]}, argv=["--dry-run"])
|
||||
assert code == 0 and _bytes() == before
|
||||
|
||||
|
||||
def test_a_removal_steam_has_undone_is_named(repo, capsys):
|
||||
with open(os.path.join("scripts", "removed_games.jsonl"), "w", encoding="utf-8") as f:
|
||||
for appid, code in (("4214820", "unavailable"), ("620", "not_free"), ("99", "admin_removed")):
|
||||
f.write(json.dumps({"appid": appid, "name": f"Old {appid}", "status_code": code,
|
||||
"removed_at": OLD}) + "\n")
|
||||
client = FakeClient(items={"730": _item("730"), "4214820": _item("4214820"),
|
||||
"620": _item("620", is_free=False), "99": _item("99")})
|
||||
repo.run([_rec("730")], client)
|
||||
out = capsys.readouterr().out
|
||||
assert "::notice::1 removed game(s)" in out and "4214820 Old 4214820" in out
|
||||
assert "620" not in out.split("::notice::")[1] and "Old 99" not in out
|
||||
|
||||
|
||||
def test_the_budget_leaves_the_rest_unasked(repo, monkeypatch):
|
||||
client = FakeClient(items={"730": _item("730", unlisted=True)})
|
||||
monkeypatch.setenv(budget.ENV, "0.000001")
|
||||
code, live = repo.run([_rec("730")], client, health={"730": [NOT_FREE]})
|
||||
assert (code, live) == (0, {"730"}) and repo.health_calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("item, why", [
|
||||
(None, "unknown"),
|
||||
({"visible": False, "is_free": True}, "hidden"),
|
||||
({"visible": True, "unlisted": True}, "unlisted"),
|
||||
({"visible": True, "is_free": True, "release": {"is_coming_soon": True}}, "coming_soon"),
|
||||
({"visible": True}, "not_free"),
|
||||
({"is_free": True, "type": 2}, None), # a free mod: type is not read
|
||||
(FREE, None),
|
||||
])
|
||||
def test_what_the_screen_flags(item, why):
|
||||
from core.store_items import screen_item
|
||||
assert screen_item(item) == why
|
||||
@@ -1,8 +1,15 @@
|
||||
"""
|
||||
refetch_all.py clears what Steam can give back and keeps what a person
|
||||
decided. Run from the repository root: python -m pytest scripts/tests
|
||||
decided, one game at a time: a game appdetails does not answer for, or the
|
||||
ones a part cut short by its budget never reached, stay exactly as they were.
|
||||
Run from the repository root: python -m pytest scripts/tests
|
||||
"""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
import refetch_all
|
||||
from core import budget, data_store
|
||||
|
||||
|
||||
def test_clear_fetchable_resets_a_default_to_the_skeleton_s_value():
|
||||
@@ -19,3 +26,93 @@ def test_clear_fetchable_keeps_a_decision():
|
||||
refetch_all.clear_fetchable(game)
|
||||
assert (game["anti_cheat"], game["anti_cheat_note"], game["is_kernel_ac"], game["genre"]) == \
|
||||
("BattlEye", "BattlEye [Kernel]", True, "FPS")
|
||||
|
||||
|
||||
# ──────────── In parts, game by game ────────────
|
||||
|
||||
|
||||
def _rec(appid: str, **fields) -> dict:
|
||||
base = {"link": f"https://store.steampowered.com/app/{appid}/", "name": f"Game {appid}",
|
||||
"description": "kept", "tags": ["FPS"], "reviews": "50% (Mixed)", "genre": "FPS",
|
||||
"anti_cheat": "-", "anti_cheat_note": "", "is_kernel_ac": None, "type_game": "online",
|
||||
"notes": "Not reviewed yet", "last_updated": "2026-01-01T00:00:00Z"}
|
||||
base.update(fields)
|
||||
return base
|
||||
|
||||
|
||||
class _Client:
|
||||
def __init__(self, failing=()):
|
||||
self.failing = set(failing)
|
||||
self.asked: list[str] = []
|
||||
|
||||
def fetch_app_details_full(self, appid):
|
||||
self.asked.append(appid)
|
||||
if appid in self.failing:
|
||||
return ("network_error", None)
|
||||
return ("ok", {"name": f"Fresh {appid}", "is_free": True})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, part", [("1/1", (1, 1)), (" 3 / 8 ", (3, 8)), ("8/8", (8, 8))])
|
||||
def test_a_part_is_read(text, part):
|
||||
assert refetch_all.parse_part(text) == part
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["0/4", "5/4", "1/0", "x", "", "1/2/3"])
|
||||
def test_a_bad_part_is_refused(text):
|
||||
with pytest.raises(Exception):
|
||||
refetch_all.parse_part(text)
|
||||
|
||||
|
||||
def test_the_parts_split_the_catalogue():
|
||||
games = [_rec(str(10 * i)) for i in range(1, 501)]
|
||||
counts = [sum(refetch_all.in_part(g, (k, 8)) for g in games) for k in range(1, 9)]
|
||||
assert sum(counts) == len(games) and min(counts) > 0
|
||||
|
||||
|
||||
def test_a_game_appdetails_does_not_answer_for_is_left_as_it_was(monkeypatch):
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(refetch_all, "fetch_full",
|
||||
lambda g, client=None, prefetched_details=None, guard=None: calls.append(g["link"]))
|
||||
game = _rec("730")
|
||||
before = dict(game)
|
||||
assert refetch_all.refetch_one(game, _Client(failing={"730"}), None) is False
|
||||
assert game == before and calls == []
|
||||
|
||||
|
||||
def test_a_game_is_cleared_and_fetched_on_a_copy(monkeypatch):
|
||||
def fake_fetch(g, client=None, prefetched_details=None, guard=None):
|
||||
assert g["description"] == "" and g["reviews"] == "N/A" # cleared before the fetch
|
||||
g["name"] = prefetched_details["name"]
|
||||
|
||||
monkeypatch.setattr(refetch_all, "fetch_full", fake_fetch)
|
||||
game = _rec("730", anti_cheat="BattlEye")
|
||||
assert refetch_all.refetch_one(game, _Client(), None) is True
|
||||
assert game["name"] == "Fresh 730" and game["anti_cheat"] == "BattlEye"
|
||||
|
||||
|
||||
def test_a_cut_part_leaves_every_other_game_alone_and_resumes_oldest_first(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
os.makedirs("data")
|
||||
games = [_rec(str(10 * i), last_updated=f"2026-01-{i:02d}T00:00:00Z") for i in range(1, 9)]
|
||||
data_store.save_main(games, apply_human_overrides=False)
|
||||
client = _Client()
|
||||
monkeypatch.setattr(refetch_all, "get_client", lambda: client)
|
||||
monkeypatch.setattr(refetch_all.OverrideGuard, "load", classmethod(lambda cls: None))
|
||||
monkeypatch.setattr(refetch_all, "STEAM_API_KEY", "")
|
||||
|
||||
clock = [0.0]
|
||||
monkeypatch.setattr(budget.time, "monotonic", lambda: clock[0])
|
||||
|
||||
def fake_fetch(g, client=None, prefetched_details=None, guard=None):
|
||||
clock[0] += 40
|
||||
g["name"] = prefetched_details["name"]
|
||||
g["last_updated"] = "2026-10-23T00:00:00Z"
|
||||
|
||||
monkeypatch.setattr(refetch_all, "fetch_full", fake_fetch)
|
||||
monkeypatch.setattr("core.fetcher.time.sleep", lambda s: None)
|
||||
monkeypatch.setenv(budget.ENV, "1")
|
||||
refetch_all.main(["--part", "1/1"])
|
||||
after = {data_store.extract_appid(g["link"]): g for g in data_store.load_main()}
|
||||
assert client.asked == ["10", "20"] # the two oldest, then the budget ran out
|
||||
assert after["10"]["name"] == "Fresh 10" and after["30"]["name"] == "Game 30"
|
||||
assert after["30"]["description"] == "kept" # never cleared
|
||||
@@ -239,3 +239,50 @@ def test_refetch_all_never_clears_it():
|
||||
# The other player fields are cleared and fetched again; this one cannot be.
|
||||
assert "all_time_peak" not in refetch_all.CLEARABLE
|
||||
assert "all_time_peak" not in refetch_all.CONDITIONAL
|
||||
|
||||
|
||||
def test_the_budget_stops_the_steamcharts_part_and_keeps_what_was_raised(monkeypatch):
|
||||
from core import budget
|
||||
|
||||
clock = [0.0]
|
||||
monkeypatch.setattr(budget.time, "monotonic", lambda: clock[0])
|
||||
deadline = budget.Deadline(1)
|
||||
games = [_game(str(i)) for i in range(1, 6)]
|
||||
|
||||
class Slow(_Client):
|
||||
def fetch(self, appid):
|
||||
clock[0] += 25
|
||||
return super().fetch(appid)
|
||||
|
||||
client = Slow({str(i): Result("ok", 100 * i) for i in range(1, 6)})
|
||||
lines: list[str] = []
|
||||
tally = rp.raise_from_steamcharts(games, client, log=lines.append, deadline=deadline)
|
||||
assert client.asked == ["1", "2", "3"]
|
||||
assert tally["raised"] == 3 and tally["skipped"] == 2
|
||||
assert any(line.startswith("::warning::") for line in lines)
|
||||
|
||||
|
||||
def test_a_larger_catalogue_is_asked_less_often_never_more():
|
||||
assert rp.cycle_for(9847, rp.PEAK_CYCLE_DAYS, rp.PEAK_DAILY) == 30
|
||||
assert rp.cycle_for(20000, rp.PEAK_CYCLE_DAYS, rp.PEAK_DAILY) == 60
|
||||
|
||||
|
||||
def test_a_run_asks_today_s_slice_and_saves(tmp_path, monkeypatch, capsys):
|
||||
import os
|
||||
|
||||
from core import budget, data_store, rotation
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
os.makedirs("data")
|
||||
monkeypatch.delenv(budget.ENV, raising=False)
|
||||
games = [_game(str(10 * i), current_players="0", peak_today="0") for i in range(1, 41)]
|
||||
data_store.save_main(games, apply_human_overrides=False)
|
||||
day = rotation.slice_of("10", rp.PEAK_CYCLE_DAYS) + 30 * 700
|
||||
client = _Client({str(10 * i): Result("ok", 1000 + i) for i in range(1, 41)})
|
||||
monkeypatch.setattr(rp, "SteamChartsClient", lambda: client)
|
||||
rp.main(["--day", str(day)])
|
||||
assert "10" in client.asked
|
||||
assert all(rotation.slice_of(a, 30) == day % 30 for a in client.asked)
|
||||
after = {data_store.extract_appid(g["link"]): g for g in data_store.load_main()}
|
||||
assert after["10"]["all_time_peak"] == "1,001"
|
||||
assert f"slice {day % 30}/30" in capsys.readouterr().out
|
||||
@@ -391,3 +391,62 @@ def test_a_blurb_that_now_fits_further_is_written_but_is_not_news():
|
||||
def test_a_rewritten_blurb_is_a_change():
|
||||
game = _game(description="Old words, cut short…")
|
||||
assert "description" in rsd.apply_app_details(game, {"short_description": "New words entirely."})
|
||||
|
||||
|
||||
# ──────────── A growing catalogue ────────────
|
||||
|
||||
def test_the_page_cycle_grows_with_the_catalogue_and_keeps_its_weekday():
|
||||
assert rsd.cycle_for(9847, rsd.PAGE_CYCLE_DAYS, rsd.PAGE_DAILY) == 7
|
||||
games = [_game(str(10 * i)) for i in range(1, 12001)]
|
||||
assert rsd.cycle_for(len(games), rsd.PAGE_CYCLE_DAYS, 1000) == 14
|
||||
for day in range(20000, 20014):
|
||||
plan = rsd.build_plan(games, day, want_details=False, page_cycle=14)
|
||||
assert len(plan) < 1000
|
||||
assert all(slice_of(rsd.extract_appid(g["link"]), 7) == day % 7 for g, _, _ in plan)
|
||||
|
||||
|
||||
def test_the_slice_comes_in_an_order_that_changes_daily():
|
||||
games = [_game(str(10 * i)) for i in range(1, 701)]
|
||||
first = [g["link"] for g, _, _ in rsd.build_plan(games, 20006, want_details=False)]
|
||||
again = [g["link"] for g, _, _ in rsd.build_plan(list(reversed(games)), 20006, want_details=False)]
|
||||
week_later = [g["link"] for g, _, _ in rsd.build_plan(games, 20013, want_details=False)]
|
||||
assert first == again
|
||||
assert sorted(first) == sorted(week_later) and first != week_later
|
||||
|
||||
|
||||
def test_a_game_appdetails_no_longer_call_free_is_listed_not_written():
|
||||
game = _game()
|
||||
before = dict(game)
|
||||
listed: list[str] = []
|
||||
details = {"name": "Counter-Strike 2", "is_free": False,
|
||||
"price_overview": {"initial": 1499, "final": 1499}}
|
||||
rsd.refresh_game(game, FakeClient(details=details), page=False, details=True, not_free=listed)
|
||||
assert listed == ["730 Counter-Strike 2"]
|
||||
assert game["notes"] == before["notes"]
|
||||
listed.clear()
|
||||
rsd.refresh_game(_game(), FakeClient(details={"name": "x", "is_free": True}), page=False, details=True,
|
||||
not_free=listed)
|
||||
assert listed == []
|
||||
|
||||
|
||||
def test_a_run_refreshes_today_s_slice_and_lists_what_appdetails_calls_paid(tmp_path, monkeypatch, capsys):
|
||||
import os
|
||||
|
||||
from core import budget, data_store
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
os.makedirs("data")
|
||||
monkeypatch.delenv(budget.ENV, raising=False)
|
||||
monkeypatch.setattr("core.fetcher.time.sleep", lambda s: None)
|
||||
games = [_game(str(10 * i), name=f"Game {i}") for i in range(1, 61)]
|
||||
data_store.save_main(games, apply_human_overrides=False)
|
||||
client = FakeClient(details={"name": "Renamed", "is_free": False, "price_overview": {"initial": 999}})
|
||||
monkeypatch.setattr(rsd, "get_client", lambda: client)
|
||||
rsd.main(["--only", "details", "--day", "20030"])
|
||||
out = capsys.readouterr().out
|
||||
asked = [c.split(":")[1] for c in client.calls]
|
||||
assert asked and all(rsd.due(_game(a), 30, 20030) for a in asked)
|
||||
assert "slice 20/30" in out
|
||||
assert "::warning::appdetails no longer calls" in out
|
||||
after = {data_store.extract_appid(g["link"]): g for g in data_store.load_main()}
|
||||
assert all(after[a]["name"] == "Renamed" for a in asked)
|
||||
@@ -132,6 +132,23 @@ def test_anticheat_items_survive_a_failed_batch(monkeypatch):
|
||||
assert sent["data_request"] == {"include_extra_details": True}
|
||||
|
||||
|
||||
def test_items_pass_the_data_request_through_and_tell_unknown_from_unanswered(monkeypatch):
|
||||
monkeypatch.setattr(steam_client, "STORE_ITEMS_BATCH", 2)
|
||||
answered = {"response": {"store_items": [
|
||||
{"id": 730, "success": 1, "reviews": {}},
|
||||
{"id": 1, "appid": 0, "success": 15}, # unknown to Steam
|
||||
]}}
|
||||
client, session = _client(_Resp(steam_client.STORE_ITEMS_URL, body=answered),
|
||||
_Resp(steam_client.STORE_ITEMS_URL, status_code=500))
|
||||
monkeypatch.setattr(steam_client, "MAX_RETRIES", 1)
|
||||
items, failed = client.fetch_items(["730", "1", "570"], {"include_reviews": True})
|
||||
assert list(items) == ["730"]
|
||||
assert failed == {"570"} # "1" was answered for: Steam does not know it
|
||||
sent = json.loads(session.requests[0]["params"]["input_json"])
|
||||
assert sent["data_request"] == {"include_reviews": True}
|
||||
assert sent["context"]["country_code"] == "US"
|
||||
|
||||
|
||||
def test_anticheat_items_give_up_after_failures_in_a_row(monkeypatch):
|
||||
monkeypatch.setattr(steam_client, "STORE_ITEMS_BATCH", 1)
|
||||
bad = [_Resp(steam_client.STORE_ITEMS_URL, status_code=403) for _ in range(2)]
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
"""
|
||||
update_reviews.py reads every game's review summary from GetItems in a
|
||||
hundred-odd requests. These pin what makes that safe to run daily: Steam's
|
||||
floored percentage is written in the stored shape, a count of 0 is never
|
||||
trusted on its own, an unanswered game keeps its value, last_updated moves
|
||||
only on a change or on the heartbeat that keeps the site's "stale" badge
|
||||
quiet, and a second run writes the same bytes.
|
||||
|
||||
Steam is never called: the client is a fake. Run from the repository root:
|
||||
python -m pytest scripts/tests
|
||||
"""
|
||||
import os
|
||||
import zlib
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
import update_reviews as ur
|
||||
from core import budget, data_store
|
||||
from core.fetcher import reviews_from_summary
|
||||
|
||||
STAMP = "2026-10-23T06:30:00Z"
|
||||
OLD = "2026-01-01T00:00:00Z"
|
||||
NOW = datetime(2026, 10, 23, 6, 30, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _recent(days: int = 1) -> str:
|
||||
return (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _rec(appid: str, reviews: str, last_updated: str = OLD) -> dict:
|
||||
return {"link": f"https://store.steampowered.com/app/{appid}/", "name": f"Game {appid}",
|
||||
"reviews": reviews, "type_game": "online", "notes": "Not reviewed yet",
|
||||
"last_updated": last_updated}
|
||||
|
||||
|
||||
def _item(appid: str, count=None, pct=None, label=None) -> dict:
|
||||
item = {"id": int(appid), "success": 1}
|
||||
if count is not None:
|
||||
summary = {"review_count": count, "review_score_label": label}
|
||||
if pct is not None:
|
||||
summary["percent_positive"] = pct
|
||||
item["reviews"] = {"summary_filtered": summary}
|
||||
return item
|
||||
|
||||
|
||||
class FakeClient:
|
||||
"""GetItems items by appid (absent: Steam does not know it), the appids
|
||||
whose batch failed, and appreviews' query_summary by appid (None: failed)."""
|
||||
|
||||
def __init__(self, items=None, failed=(), appreviews=None):
|
||||
self.items, self.failed = items or {}, set(failed)
|
||||
self.appreviews = appreviews or {}
|
||||
self.asked: list[str] = []
|
||||
|
||||
def fetch_items(self, appids, data_request, give_up_after=10):
|
||||
assert data_request == {"include_reviews": True}
|
||||
return ({a: self.items[a] for a in appids if a in self.items and a not in self.failed},
|
||||
{a for a in appids if a in self.failed})
|
||||
|
||||
def fetch_reviews(self, appid):
|
||||
self.asked.append(appid)
|
||||
return self.appreviews.get(appid)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
os.makedirs("data")
|
||||
monkeypatch.setattr(ur, "now_iso", lambda: STAMP)
|
||||
monkeypatch.delenv(budget.ENV, raising=False)
|
||||
monkeypatch.setattr("core.fetcher.time.sleep", lambda s: None)
|
||||
|
||||
def run(records=None, client=None, argv=()):
|
||||
if records is not None:
|
||||
data_store.save_main(records, apply_human_overrides=False)
|
||||
monkeypatch.setattr(ur, "get_client", lambda: client)
|
||||
ur.main(list(argv))
|
||||
return {data_store.extract_appid(g["link"]): g for g in data_store.load_main()}
|
||||
return run
|
||||
|
||||
|
||||
def _bytes() -> dict:
|
||||
return {n: open(os.path.join("data", n), "rb").read()
|
||||
for n in sorted(os.listdir("data")) if n.endswith((".jsonl", ".json"))}
|
||||
|
||||
|
||||
# ──────────── The stored shape ────────────
|
||||
|
||||
@pytest.mark.parametrize("pos, total, expected", [
|
||||
(29, 100, "29% (Mixed)"), # floor(29/100*100) in floating point is 28
|
||||
(2, 3, "66% (Mixed)"),
|
||||
(999, 1000, "99% (Mixed)"),
|
||||
(0, 0, "No reviews"),
|
||||
])
|
||||
def test_appreviews_is_floored_in_integers(pos, total, expected):
|
||||
summary = {"total_positive": pos, "total_reviews": total, "review_score_desc": "Mixed"}
|
||||
assert reviews_from_summary(summary) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stored, item, expected", [
|
||||
("86% (Very Positive)", _item("1", 9943066, 85, "Very Positive"), (ur.SET, "85% (Very Positive)")),
|
||||
("N/A", _item("1", 3, 100, "3 user reviews"), (ur.SET, "100% (3 user reviews)")),
|
||||
("N/A", _item("1", 12, 50, ""), (ur.SET, "50%")),
|
||||
("No reviews", _item("1", 0, 0, "No user reviews"), (ur.KEEP, None)),
|
||||
("93% (Very Positive)", _item("1", 0, 0, "No user reviews"), (ur.ASK, None)),
|
||||
("N/A", _item("1", 0, 0, "No user reviews"), (ur.ASK, None)),
|
||||
("", _item("1", 0), (ur.ASK, None)),
|
||||
("50% (Mixed)", None, (ur.UNKNOWN, None)),
|
||||
("50% (Mixed)", _item("1"), (ur.UNKNOWN, None)), # no summary
|
||||
("50% (Mixed)", _item("1", 40, None, "Mixed"), (ur.UNKNOWN, None)), # count without a share
|
||||
("50% (Mixed)", _item("1", 40, 101, "Mixed"), (ur.UNKNOWN, None)),
|
||||
("50% (Mixed)", {"reviews": {"summary_filtered": {"percent_positive": 50}}}, (ur.UNKNOWN, None)),
|
||||
])
|
||||
def test_what_an_item_says_to_do(stored, item, expected):
|
||||
assert ur.decide(stored, item) == expected
|
||||
|
||||
|
||||
def test_the_heartbeat_spreads_over_a_week_and_reads_a_bad_stamp_as_due():
|
||||
game = _rec("730", "85% (Very Positive)")
|
||||
days = ur.HEARTBEAT_DAYS + zlib.crc32(b"730") % ur.HEARTBEAT_JITTER_DAYS
|
||||
assert 14 <= days <= 20
|
||||
game["last_updated"] = (NOW - timedelta(days=days - 1)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
assert not ur.heartbeat_due(game, NOW)
|
||||
game["last_updated"] = (NOW - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
assert ur.heartbeat_due(game, NOW)
|
||||
game["last_updated"] = ""
|
||||
assert ur.heartbeat_due(game, NOW)
|
||||
|
||||
|
||||
# ──────────── A run ────────────
|
||||
|
||||
def test_a_changed_summary_is_written_and_stamped(repo):
|
||||
client = FakeClient(items={"730": _item("730", 100, 85, "Very Positive")})
|
||||
out = repo([_rec("730", "86% (Very Positive)", _recent())], client)
|
||||
assert out["730"]["reviews"] == "85% (Very Positive)"
|
||||
assert out["730"]["last_updated"] == STAMP
|
||||
assert client.asked == []
|
||||
|
||||
|
||||
def test_an_unchanged_game_keeps_its_stamp_until_its_heartbeat(repo):
|
||||
recent, old = _recent(), _recent(days=40)
|
||||
client = FakeClient(items={"730": _item("730", 100, 85, "Very Positive"),
|
||||
"570": _item("570", 100, 80, "Very Positive")})
|
||||
out = repo([_rec("730", "85% (Very Positive)", recent), _rec("570", "80% (Very Positive)", old)], client)
|
||||
assert out["730"]["last_updated"] == recent
|
||||
assert out["570"]["last_updated"] == STAMP # read, and due its heartbeat
|
||||
|
||||
|
||||
def test_a_second_run_writes_the_same_bytes(repo):
|
||||
client = FakeClient(items={"730": _item("730", 100, 85, "Very Positive"),
|
||||
"570": _item("570", 0, 0, "No user reviews")},
|
||||
appreviews={"570": {"total_reviews": 4, "total_positive": 3,
|
||||
"review_score_desc": "4 user reviews"}})
|
||||
repo([_rec("730", "86% (Very Positive)"), _rec("570", "N/A")], client)
|
||||
first = _bytes()
|
||||
out = repo(None, client)
|
||||
assert _bytes() == first
|
||||
assert out["570"]["reviews"] == "75% (4 user reviews)"
|
||||
|
||||
|
||||
def test_a_zero_is_checked_on_appreviews_before_a_percentage_goes(repo):
|
||||
zero = _item("845070", 0, 0, "No user reviews")
|
||||
failing = FakeClient(items={"845070": zero}, appreviews={"845070": None})
|
||||
out = repo([_rec("845070", "93% (Very Positive)")], failing)
|
||||
assert out["845070"]["reviews"] == "93% (Very Positive)"
|
||||
assert out["845070"]["last_updated"] == OLD # not read: no heartbeat either
|
||||
assert failing.asked == ["845070"]
|
||||
|
||||
answered = FakeClient(items={"845070": zero}, appreviews={"845070": {
|
||||
"total_reviews": 5843, "total_positive": 5318, "review_score_desc": "Very Positive"}})
|
||||
assert repo(None, answered)["845070"]["reviews"] == "91% (Very Positive)"
|
||||
|
||||
gone = FakeClient(items={"845070": zero}, appreviews={"845070": {"total_reviews": 0}})
|
||||
assert repo(None, gone)["845070"]["reviews"] == "No reviews"
|
||||
|
||||
|
||||
def test_no_reviews_stays_without_asking(repo):
|
||||
client = FakeClient(items={"4263530": _item("4263530", 0, 0, "No user reviews")})
|
||||
out = repo([_rec("4263530", "No reviews", _recent())], client)
|
||||
assert out["4263530"]["reviews"] == "No reviews" and client.asked == []
|
||||
|
||||
|
||||
def test_an_unanswered_game_keeps_everything(repo):
|
||||
client = FakeClient(items={"730": _item("730", 100, 85, "Very Positive"),
|
||||
"570": _item("570", 100, 70, "Mostly Positive")}, failed={"570"})
|
||||
out = repo([_rec("730", "86% (Very Positive)"), _rec("570", "80% (Very Positive)")], client)
|
||||
assert out["570"]["reviews"] == "80% (Very Positive)" and out["570"]["last_updated"] == OLD
|
||||
assert out["730"]["reviews"] == "85% (Very Positive)"
|
||||
|
||||
|
||||
def test_no_answer_at_all_writes_nothing(repo):
|
||||
repo([_rec("730", "86% (Very Positive)"), _rec("570", "80% (Very Positive)")], FakeClient())
|
||||
before = _bytes()
|
||||
repo(None, FakeClient(failed={"730", "570"}))
|
||||
assert _bytes() == before
|
||||
|
||||
|
||||
def test_appreviews_is_capped_and_stops_at_the_deadline(repo, monkeypatch):
|
||||
zero = {a: _item(a, 0, 0, "No user reviews") for a in ("10", "20", "30")}
|
||||
summary = {"total_reviews": 1, "total_positive": 1, "review_score_desc": "1 user reviews"}
|
||||
client = FakeClient(items=zero, appreviews={a: summary for a in zero})
|
||||
monkeypatch.setattr(ur, "ASK_CAP", 2)
|
||||
out = repo([_rec(a, "N/A") for a in zero], client)
|
||||
assert len(client.asked) == 2
|
||||
assert sum(g["reviews"] == "100% (1 user reviews)" for g in out.values()) == 2
|
||||
|
||||
client = FakeClient(items=zero, appreviews={a: summary for a in zero})
|
||||
monkeypatch.setenv(budget.ENV, "0.000001")
|
||||
repo(None, client)
|
||||
assert client.asked == []
|
||||
|
||||
|
||||
def test_a_dry_run_writes_nothing(repo):
|
||||
repo([_rec("730", "86% (Very Positive)")], FakeClient())
|
||||
before = _bytes()
|
||||
repo(None, FakeClient(items={"730": _item("730", 100, 85, "Very Positive")}), argv=["--dry-run"])
|
||||
assert _bytes() == before
|
||||
@@ -12,6 +12,7 @@ Output: games/top-offline.md (10 columns, same shape as top-online.md).
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from datetime import datetime, timezone
|
||||
from core.budget import Deadline
|
||||
from core.constants import STEAM_API_KEY, GAMES_DIR, TOP_OFFLINE_LIMIT
|
||||
from core.data_store import load_main, save_main, extract_appid
|
||||
from core.fetcher import update_players_only
|
||||
@@ -58,6 +59,7 @@ def main():
|
||||
if not STEAM_API_KEY:
|
||||
print("No Steam API key (set STEAM_API_KEY).")
|
||||
sys.exit(1)
|
||||
deadline = Deadline.from_env()
|
||||
games = load_main()
|
||||
offline = [g for g in games if _is_offline(g) and g.get("status", "active") == "active"]
|
||||
if not offline:
|
||||
@@ -68,7 +70,7 @@ def main():
|
||||
for g in offline:
|
||||
aid = extract_appid(g.get("link", ""))
|
||||
if aid: prev[aid] = _ppc(g.get("current_players", "0"))
|
||||
update_players_only(games, type_filter="offline")
|
||||
update_players_only(games, type_filter="offline", deadline=deadline)
|
||||
# Re-filter after update (status may have changed).
|
||||
offline = [g for g in games if _is_offline(g) and g.get("status", "active") == "active"]
|
||||
offline.sort(key=lambda g: _ppc(g.get("current_players", "0")), reverse=True)
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from datetime import datetime, timezone
|
||||
from core.budget import Deadline
|
||||
from core.constants import STEAM_API_KEY, GAMES_DIR, TOP_ONLINE_LIMIT
|
||||
from core.data_store import load_main, save_main, extract_appid
|
||||
from core.fetcher import update_players_only
|
||||
@@ -39,6 +40,7 @@ def _stars(r):
|
||||
|
||||
def main():
|
||||
if not STEAM_API_KEY: print("No key."); sys.exit(1)
|
||||
deadline = Deadline.from_env()
|
||||
games = load_main()
|
||||
online = [g for g in games if g.get("type_game","").lower()=="online"
|
||||
and g.get("status","active")=="active"]
|
||||
@@ -48,7 +50,7 @@ def main():
|
||||
for g in online:
|
||||
aid = extract_appid(g.get("link",""))
|
||||
if aid: prev[aid] = _ppc(g.get("current_players","0"))
|
||||
update_players_only(games)
|
||||
update_players_only(games, deadline=deadline)
|
||||
# Re-filter after update (status may have changed)
|
||||
online = [g for g in games if g.get("type_game","").lower()=="online"
|
||||
and g.get("status","active")=="active"]
|
||||
|
||||
@@ -2,14 +2,16 @@
|
||||
"""Full data update – enriches incomplete records."""
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from core.budget import Deadline
|
||||
from core.data_store import load_main, save_main, migrate_record
|
||||
from core.fetcher import update_all_full
|
||||
|
||||
def main():
|
||||
deadline = Deadline.from_env()
|
||||
games = load_main()
|
||||
print(f"Loaded {len(games)} games")
|
||||
for g in games: migrate_record(g)
|
||||
update_all_full(games)
|
||||
update_all_full(games, deadline=deadline)
|
||||
save_main(games)
|
||||
print(f"\n✓ Saved {len(games)} games")
|
||||
|
||||
|
||||
+175
-10
@@ -1,15 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Update reviews only. Every 2 days."""
|
||||
import sys, os
|
||||
"""
|
||||
Refresh every game's review summary from Steam's own, a hundred-odd requests a day.
|
||||
|
||||
This used to ask store.steampowered.com/appreviews about one game at a time:
|
||||
two seconds a game, five and a half hours at 9,800 games, and on 2026-10-23 it
|
||||
ran into GitHub's six-hour limit and was stopped with nothing saved.
|
||||
IStoreBrowseService/GetItems answers for fifty games a request with the summary
|
||||
the store shows (core/store_items.review_summary): the whole catalogue in about
|
||||
two minutes.
|
||||
|
||||
The rules:
|
||||
- A summary with reviews is written "85% (Very Positive)": Steam's floored
|
||||
percentage and its label, the vocabulary appreviews has ("3 user
|
||||
reviews", "Mixed").
|
||||
- A count of 0 is never written on GetItems' word alone: its summary has
|
||||
said 0 for a game with thousands of reviews. A game stored as "No
|
||||
reviews" stays so; one stored with a percentage, "N/A" or nothing is asked
|
||||
on appreviews instead (at most ASK_CAP a run, in an order that changes
|
||||
daily), and that answer is written, "No reviews" included.
|
||||
- No item, no usable summary, a failed batch or a failed appreviews request
|
||||
leaves the stored value alone. When GetItems answers for no game at all,
|
||||
nothing is written.
|
||||
- last_updated moves when the value changes, and on a heartbeat: a game
|
||||
whose summary was read and whose stamp is 14 to 20 days old (crc32
|
||||
jitter, so the stamps spread out) is stamped again. The site calls a
|
||||
record older than 30 days stale (web/src/lib/validation.ts, /health),
|
||||
and this job used to stamp every game on every run; the heartbeat keeps
|
||||
that meaning - read by the pipeline lately - without rewriting every
|
||||
record every day.
|
||||
- Everything is written by save_main().
|
||||
|
||||
Usage (from the repository root):
|
||||
python scripts/update_reviews.py
|
||||
python scripts/update_reviews.py --limit 200 --dry-run
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import zlib
|
||||
from collections import Counter
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from core.data_store import load_main, save_main
|
||||
from core.fetcher import update_reviews_only
|
||||
|
||||
def main():
|
||||
from core.budget import Deadline
|
||||
from core.data_store import extract_appid, load_main, migrate_record, now_iso, save_main
|
||||
from core.fetcher import process_batch, reviews_from_summary, reviews_text
|
||||
from core.normalize import normalize_records
|
||||
from core.rotation import daily_order, day_number
|
||||
from core.steam_client import get_client
|
||||
from core.store_items import review_summary
|
||||
|
||||
ASK_CAP = 300
|
||||
HEARTBEAT_DAYS = 14
|
||||
HEARTBEAT_JITTER_DAYS = 7
|
||||
NO_REVIEWS = "No reviews"
|
||||
|
||||
# What decide() says to do with one game.
|
||||
SET = "set" # write this value
|
||||
ASK = "ask" # ask appreviews: GetItems said 0
|
||||
KEEP = "keep" # read, and nothing to write
|
||||
UNKNOWN = "unknown" # not read: no item or no usable summary
|
||||
|
||||
|
||||
def decide(stored: str, item: Optional[dict]) -> tuple[str, Optional[str]]:
|
||||
"""What GetItems' item says to do with a game whose reviews are `stored`."""
|
||||
found = review_summary(item)
|
||||
if found is None:
|
||||
return UNKNOWN, None
|
||||
count, pct, label = found
|
||||
if count > 0:
|
||||
return SET, reviews_text(count, pct, label)
|
||||
return (KEEP, None) if stored == NO_REVIEWS else (ASK, None)
|
||||
|
||||
|
||||
def heartbeat_due(game: dict, now: datetime) -> bool:
|
||||
"""Whether a game read today is due a new stamp although nothing changed:
|
||||
its last_updated is 14 to 20 days old (by its appid), or unreadable."""
|
||||
appid = extract_appid(game.get("link", "")) or ""
|
||||
days = HEARTBEAT_DAYS + zlib.crc32(appid.encode()) % HEARTBEAT_JITTER_DAYS
|
||||
try:
|
||||
stamp = datetime.strptime(game.get("last_updated") or "", "%Y-%m-%dT%H:%M:%SZ")
|
||||
except ValueError:
|
||||
return True
|
||||
return now - stamp.replace(tzinfo=timezone.utc) >= timedelta(days=days)
|
||||
|
||||
|
||||
def record(game: dict, value: Optional[str], now: datetime, tally: Counter) -> None:
|
||||
"""A game whose summary was read: write `value` (None: nothing new) and
|
||||
stamp it as the rules say."""
|
||||
if value is not None and game.get("reviews") != value:
|
||||
game["reviews"] = value
|
||||
game["last_updated"] = now_iso()
|
||||
tally["changed"] += 1
|
||||
elif heartbeat_due(game, now):
|
||||
game["last_updated"] = now_iso()
|
||||
tally["heartbeat"] += 1
|
||||
|
||||
|
||||
def refresh(games: list[dict], client, deadline: Optional[Deadline] = None,
|
||||
day: Optional[int] = None, now: Optional[datetime] = None) -> Counter:
|
||||
"""Refresh the reviews of `games` in place. Returns a tally: changed,
|
||||
heartbeat, unknown, unanswered (batches GetItems failed), asked,
|
||||
ask_failed, ask_skipped (over the cap or past the deadline), and
|
||||
no_answer when GetItems answered for nobody."""
|
||||
now = now or datetime.now(timezone.utc)
|
||||
by_id = {}
|
||||
for game in games:
|
||||
appid = extract_appid(game.get("link", ""))
|
||||
if appid:
|
||||
by_id[appid] = game
|
||||
items, failed = client.fetch_items(list(by_id), {"include_reviews": True})
|
||||
tally: Counter = Counter(unanswered=len(failed))
|
||||
if by_id and len(failed) == len(by_id):
|
||||
tally["no_answer"] = 1
|
||||
return tally
|
||||
|
||||
asks: list[dict] = []
|
||||
for appid, game in by_id.items():
|
||||
if appid in failed:
|
||||
continue
|
||||
kind, value = decide(game.get("reviews", ""), items.get(appid))
|
||||
if kind == UNKNOWN:
|
||||
tally["unknown"] += 1
|
||||
elif kind == ASK:
|
||||
asks.append(game)
|
||||
else:
|
||||
record(game, value, now, tally)
|
||||
|
||||
def ask(game: dict) -> None:
|
||||
summary = client.fetch_reviews(extract_appid(game["link"]))
|
||||
if summary is None:
|
||||
tally["ask_failed"] += 1
|
||||
else:
|
||||
record(game, reviews_from_summary(summary), now, tally)
|
||||
|
||||
asks = daily_order(asks, day_number() if day is None else day)
|
||||
done = process_batch(asks[:ASK_CAP], ask, "Reviews from appreviews", deadline=deadline) if asks else 0
|
||||
tally["asked"] = done
|
||||
tally["ask_skipped"] = len(asks) - done
|
||||
return tally
|
||||
|
||||
|
||||
def main(argv: Optional[list[str]] = None) -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0].strip())
|
||||
ap.add_argument("--limit", type=int, help="at most this many games (for trying it out)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="ask Steam and report, write nothing")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
deadline = Deadline.from_env()
|
||||
games = load_main()
|
||||
print(f"{len(games)} games – updating reviews...")
|
||||
update_reviews_only(games)
|
||||
save_main(games)
|
||||
print(f"✓ Done.")
|
||||
for g in games:
|
||||
migrate_record(g)
|
||||
# Compared as stored, like the other refresh jobs.
|
||||
normalize_records(games)
|
||||
targets = games if args.limit is None else games[: args.limit]
|
||||
print(f"{len(games)} games; reading the review summaries of {len(targets)} from Steam")
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
tally = refresh(targets, get_client(), deadline=deadline)
|
||||
if tally["no_answer"]:
|
||||
print("::warning::GetItems answered for no game: nothing written.")
|
||||
return
|
||||
print(f"\nReviews: {tally['changed']} changed, {tally['heartbeat']} re-stamped (heartbeat), "
|
||||
f"{tally['unknown']} without a usable summary; appreviews asked about {tally['asked']} "
|
||||
f"({tally['ask_failed']} failed, {tally['ask_skipped']} left for another day)")
|
||||
if tally["unanswered"]:
|
||||
print(f"::warning::GetItems did not answer for {tally['unanswered']} games; "
|
||||
f"they keep their reviews until the next run.")
|
||||
|
||||
if args.dry_run:
|
||||
print("Dry run: nothing written.")
|
||||
return
|
||||
save_main(games)
|
||||
print(f"✓ Saved {len(games)} games")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -194,7 +194,7 @@
|
||||
</Button>
|
||||
<p class="text-xs text-muted-foreground">
|
||||
A batch waits while the previous one is still in scripts/temp_info.jsonl, or while a data job is running or
|
||||
waiting its turn - pushing then would cancel that job.
|
||||
waiting its turn - the ingest it starts would only queue behind that job.
|
||||
</p>
|
||||
</article>
|
||||
|
||||
|
||||
@@ -71,8 +71,8 @@
|
||||
<div>
|
||||
<h1 class="text-2xl font-semibold tracking-tight">Pipeline</h1>
|
||||
<p class="max-w-3xl text-sm text-muted-foreground">
|
||||
The data workflows on GitHub Actions. Those marked data-write take turns: one runs, at most one waits, and a newer
|
||||
one cancels the waiting run.
|
||||
The data workflows on GitHub Actions. Those marked data-write take turns: one runs and the others wait in a queue,
|
||||
first in, first out.
|
||||
</p>
|
||||
</div>
|
||||
<Button variant="outline" size="sm" onclick={() => load(true)} disabled={loading}>
|
||||
|
||||
@@ -1,7 +1,18 @@
|
||||
/**
|
||||
* DATA_WORKFLOWS must name every workflow in `concurrency: data-write`:
|
||||
* auto-approve waits for exactly those, and one missing from the list is one
|
||||
* whose pending run an automatic batch could evict.
|
||||
* auto-approve waits for exactly those.
|
||||
*
|
||||
* And every one of them must:
|
||||
* - queue (`queue: max`, `cancel-in-progress: false`). With the default a
|
||||
* run that queued behind a running job evicted the one already waiting:
|
||||
* Update Reviews lost five of ten runs that way in October 2026. Mixing
|
||||
* the two in one group is not documented, so none may;
|
||||
* - check out `main`. A run's commit is fixed when the run is created, and a
|
||||
* run that waited behind another data job would write shards from before
|
||||
* that job's commit - Auto Update JSON failed four days in ten on the
|
||||
* rebase conflict in data/index.json;
|
||||
* - declare `timeout-minutes`, and a JOB_BUDGET_MINUTES at least 20 below
|
||||
* it (scripts/core/budget.py): the script stops in time to save and push.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||||
@@ -11,6 +22,26 @@ import { DATA_WORKFLOWS } from "./pipeline";
|
||||
const REPO = join(__dirname, "..", "..");
|
||||
const DIR = join(REPO, ".github", "workflows");
|
||||
|
||||
/** The YAML without its comment lines, which quote the old settings. */
|
||||
function code(file: string): string {
|
||||
return readFileSync(join(REPO, file), "utf-8")
|
||||
.split("\n")
|
||||
.filter((line) => !/^\s*#/.test(line))
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
const TIMEOUT = /timeout-minutes\s*:\s*(.+)/;
|
||||
const BUDGET = /JOB_BUDGET_MINUTES\s*:\s*(.+)/;
|
||||
|
||||
/** The whole numbers on the first line `key` matches: [60] for "60", [340, 45]
|
||||
* for "${{ inputs.backfill && 340 || 45 }}". */
|
||||
function numbers(text: string, key: RegExp): number[] {
|
||||
const line = key.exec(text);
|
||||
return line ? (line[1].match(/\d+/g) ?? []).map(Number) : [];
|
||||
}
|
||||
|
||||
const dataWrite = DATA_WORKFLOWS.filter((w) => w.dataWrite).map((w) => w.file);
|
||||
|
||||
describe("DATA_WORKFLOWS", () => {
|
||||
it("marks exactly the data-write workflows", () => {
|
||||
const inGroup = readdirSync(DIR)
|
||||
@@ -27,3 +58,35 @@ describe("DATA_WORKFLOWS", () => {
|
||||
expect(new Set(DATA_WORKFLOWS.map((w) => w.file)).size).toBe(DATA_WORKFLOWS.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe("every data-write workflow", () => {
|
||||
it("queues its runs instead of evicting the one waiting", () => {
|
||||
for (const file of dataWrite) {
|
||||
const text = code(file);
|
||||
expect(text, file).toMatch(/\bqueue\s*:\s*max\b/);
|
||||
expect(text, file).toMatch(/\bcancel-in-progress\s*:\s*false\b/);
|
||||
}
|
||||
});
|
||||
|
||||
it("checks out main, not the commit its run was created on", () => {
|
||||
for (const file of dataWrite) {
|
||||
const text = code(file);
|
||||
const checkouts = text.match(/uses\s*:\s*actions\/checkout@/g) ?? [];
|
||||
const refs = text.match(/\bref\s*:\s*main\b/g) ?? [];
|
||||
expect(checkouts.length, file).toBeGreaterThan(0);
|
||||
expect(refs.length, file).toBe(checkouts.length);
|
||||
}
|
||||
});
|
||||
|
||||
it("stops at its budget in time to save before its timeout", () => {
|
||||
for (const file of dataWrite) {
|
||||
const text = code(file);
|
||||
const timeouts = numbers(text, TIMEOUT);
|
||||
const budgets = numbers(text, BUDGET);
|
||||
expect(timeouts.length, file).toBeGreaterThan(0);
|
||||
if (budgets.length === 0) continue;
|
||||
expect(budgets.length, file).toBe(timeouts.length);
|
||||
budgets.forEach((b, i) => expect(b + 20, `${file}: budget ${b}, timeout ${timeouts[i]}`).toBeLessThanOrEqual(timeouts[i]));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -3,9 +3,9 @@
|
||||
* the GitHub API), so renaming a workflow does not break the page.
|
||||
*
|
||||
* `dataWrite` marks the workflows in `concurrency: {group: data-write}`.
|
||||
* Auto-approve commits only while none of them is running or waiting: the
|
||||
* group keeps ONE pending run, and the ingest an approval starts would evict a
|
||||
* job already waiting (pipeline.test.ts holds this list to the YAML).
|
||||
* Auto-approve commits only while none of them is running or waiting, so the
|
||||
* ingest an approval starts can begin at once (pipeline.test.ts holds this
|
||||
* list to the YAML, and every one of them to `queue: max`).
|
||||
*
|
||||
* Pure: no DOM, no Env, no I/O.
|
||||
*/
|
||||
@@ -24,7 +24,6 @@ export const DATA_WORKFLOWS: DataWorkflow[] = [
|
||||
{ file: ".github/workflows/update-reviews.yml", label: "Update reviews", dataWrite: true },
|
||||
{ file: ".github/workflows/top-online.yml", label: "Top online", dataWrite: true },
|
||||
{ file: ".github/workflows/top-offline.yml", label: "Top offline", dataWrite: true },
|
||||
{ file: ".github/workflows/check-dead-links.yml", label: "Check dead links", dataWrite: true },
|
||||
{ file: ".github/workflows/mark-dead-games.yml", label: "Mark dead games", dataWrite: true },
|
||||
{ file: ".github/workflows/purge-unhealthy.yml", label: "Purge unhealthy games", dataWrite: true },
|
||||
{ file: ".github/workflows/snapshot-daily.yml", label: "Daily snapshot", dataWrite: true },
|
||||
|
||||
@@ -17,9 +17,10 @@
|
||||
* ingested. Checked again atomically inside the commit
|
||||
* (requireEmptyQueue). Not "no row is approved": a network-skipped row
|
||||
* stays approved for 12 hours and would stall everything that long;
|
||||
* - no data-write workflow is running or waiting (GitHub Actions). That
|
||||
* group keeps ONE pending run, and the ingest this commit starts would
|
||||
* evict a job already waiting its turn;
|
||||
* - no data-write workflow is running or waiting (GitHub Actions). The
|
||||
* group queues its runs (queue: max), so the ingest this commit starts
|
||||
* would no longer evict a job waiting its turn - it would wait behind it.
|
||||
* The gate stays so that a batch lands when the ingest can start at once;
|
||||
* - something is eligible: pending, free, app_type game, health ok, old
|
||||
* enough, from an allowed source, never decided, its game not committed,
|
||||
* not in data/blocklist.jsonl.
|
||||
|
||||
Reference in new issue
Block a user