diff --git a/.github/workflows/ingest-new.yml b/.github/workflows/ingest-new.yml index d54a2b28..943d6713 100644 --- a/.github/workflows/ingest-new.yml +++ b/.github/workflows/ingest-new.yml @@ -5,26 +5,31 @@ on : # fires the ingest on a detached HEAD, and dies with exit 128 on push. # # data/blocklist.jsonl is here too: this job also applies removals - # (bash/ingest.sh runs scripts/remove_games.py first). A separate removal - # workflow would share `data-write` with this one, and each would evict the - # other's pending run. + # (bash/ingest.sh runs scripts/remove_games.py first), so one run does both. push : { branches : [ main ], paths : [ 'scripts/temp_info.jsonl', 'data/blocklist.jsonl' ] } - # Orphan recovery, not a poll. `data-write` keeps at most ONE run pending and - # cancels the older one, so a push here that queues behind a ~115-minute data - # job is silently evicted the moment any OTHER data-write workflow queues -- - # and nothing retries it, leaving approved games sitting in temp_info.jsonl - # until the next unrelated push. Re-running on a clear file is free: - # ingest_new.py returns immediately and `git diff --staged --quiet` skips the - # commit, so this is idempotent by construction. + # A safety net, not a poll. Before data-write queued its runs (queue: max), + # a push here that waited behind a long data job was evicted the moment any + # other data-write workflow queued, and approved games sat in + # temp_info.jsonl until the next unrelated push. Nothing is evicted now, but + # a failed run still leaves its links queued. Re-running on a clear file is + # free: ingest_new.py returns immediately and `git diff --staged --quiet` + # skips the commit, so this is idempotent by construction. schedule : [ { cron : '0 */3 * * *' } ] workflow_dispatch : permissions : { contents : write } +# queue: max - a run that finds another data job running waits its turn +# (first in, first out) instead of evicting the run already waiting. concurrency : group : data-write cancel-in-progress : false + queue : max jobs : ingest : runs-on : ubuntu-latest + # About seven seconds a game, and an /admin batch holds at most 100. No + # time budget: ingest_new.py empties the whole queue file when it is done, + # so a game it had not reached would be lost. + timeout-minutes : 120 steps : - uses : actions/checkout@v6 diff --git a/.github/workflows/mark-dead-games.yml b/.github/workflows/mark-dead-games.yml index 4a0d1021..563a0d69 100644 --- a/.github/workflows/mark-dead-games.yml +++ b/.github/workflows/mark-dead-games.yml @@ -19,13 +19,17 @@ permissions: contents: write actions: write +# queue: max - a run that finds another data job running waits its turn +# (first in, first out) instead of evicting the run already waiting. concurrency: group: data-write cancel-in-progress: false + queue: max jobs: mark: runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 # The tip of main when the job starts, not when it was queued: a run diff --git a/.github/workflows/snapshot-daily.yml b/.github/workflows/snapshot-daily.yml index 19d9ea91..ab4173d6 100644 --- a/.github/workflows/snapshot-daily.yml +++ b/.github/workflows/snapshot-daily.yml @@ -4,15 +4,22 @@ on: workflow_dispatch: permissions: {contents: write} # Shares the data-write group with every other workflow that commits to -# data/ — without it this can push mid-run against a 2-hour job. +# data/ — without it this can push mid-run against a long job. queue: max - +# a run that finds another data job running waits its turn (first in, first +# out) instead of evicting the run already waiting. concurrency: group: data-write cancel-in-progress: false + queue: max jobs: snapshot: runs-on: ubuntu-latest + timeout-minutes: 20 steps: + # main as it is when the job STARTS, so the snapshot holds what the + # jobs before it committed. - uses: actions/checkout@v4 + with: {ref: main} - uses: actions/setup-python@v5 with: {python-version: '3.15'} - run: chmod +x ./bash/snapshot.sh && ./bash/snapshot.sh diff --git a/.github/workflows/top-offline.yml b/.github/workflows/top-offline.yml index 8b86e873..2e245344 100644 --- a/.github/workflows/top-offline.yml +++ b/.github/workflows/top-offline.yml @@ -5,15 +5,27 @@ on: schedule: [{cron: '0 5 1,15 * *'}] workflow_dispatch: permissions: {contents: write} +# queue: max - a run that finds another data job running waits its turn +# (first in, first out) instead of evicting the run already waiting. concurrency: group: data-write cancel-in-progress: false + queue: max jobs: leaderboard: runs-on: ubuntu-latest + # About 0.55 s a game through the keyed Web API: ~65 minutes for 7,000 + # offline games, ~2 hours at 14,000. The budget stops the sampling with + # time left to save and commit. + timeout-minutes: 240 steps: + # main as it is when the job STARTS: a run that waited its turn must + # work on what the job before it committed. - uses: actions/checkout@v4 + with: {ref: main} - uses: actions/setup-python@v5 with: {python-version: '3.15'} - run: chmod +x ./bash/offline.sh && ./bash/offline.sh - env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'} + env: + STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}' + JOB_BUDGET_MINUTES: '200' diff --git a/.github/workflows/top-online.yml b/.github/workflows/top-online.yml index 1deb0210..d6b86837 100644 --- a/.github/workflows/top-online.yml +++ b/.github/workflows/top-online.yml @@ -3,15 +3,27 @@ on: schedule: [{cron: '0 3 * * 0,3,5'}] workflow_dispatch: permissions: {contents: write} +# queue: max - a run that finds another data job running waits its turn +# (first in, first out) instead of evicting the run already waiting. concurrency: group: data-write cancel-in-progress: false + queue: max jobs: leaderboard: runs-on: ubuntu-latest + # The player counts come from the keyed Web API with no pause between + # batches, about 0.55 s a game: ~27 minutes for 2,900 online games. The + # budget stops the sampling with time left to save and commit. + timeout-minutes: 150 steps: + # main as it is when the job STARTS: a run that waited its turn must + # work on what the job before it committed. - uses: actions/checkout@v4 + with: {ref: main} - uses: actions/setup-python@v5 with: {python-version: '3.15'} - run: chmod +x ./bash/online.sh && ./bash/online.sh - env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'} + env: + STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}' + JOB_BUDGET_MINUTES: '120' diff --git a/.github/workflows/update-json.yml b/.github/workflows/update-json.yml index b3d758e6..18b3b48f 100644 --- a/.github/workflows/update-json.yml +++ b/.github/workflows/update-json.yml @@ -3,15 +3,26 @@ on: schedule: [{cron: '0 0 * * *'}] workflow_dispatch: permissions: {contents: write} +# queue: max - a run that finds another data job running waits its turn +# (first in, first out) instead of evicting the run already waiting. concurrency: group: data-write cancel-in-progress: false + queue: max jobs: update: runs-on: ubuntu-latest + timeout-minutes: 90 steps: + # main as it is when the job STARTS. The commit the scheduled run was + # created on is older whenever another data job ran first, and this one + # then failed on a conflict in data/index.json (four times in ten days, + # October 2026). - uses: actions/checkout@v6 + with: {ref: main} - uses: actions/setup-python@v6 with: {python-version: '3.15'} - run: chmod +x ./bash/json.sh && ./bash/json.sh - env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'} + env: + STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}' + JOB_BUDGET_MINUTES: '60' diff --git a/bash/ingest.sh b/bash/ingest.sh index 4922ebd8..5e241973 100755 --- a/bash/ingest.sh +++ b/bash/ingest.sh @@ -3,9 +3,10 @@ # # remove_games.py applies data/blocklist.jsonl (games the maintainer removed in # /admin or with delete_game.py) before ingest_new.py adds the queued links. -# One workflow does both on purpose: `data-write` keeps ONE pending run and a -# newer one evicts it, so a separate removal workflow and the ingest would keep -# cancelling each other. Whichever run survives does both. +# One workflow does both: a commit to either file starts one run that settles +# the two, in that order. (It was also the only way to do it while +# `data-write` kept a single pending run and a separate removal workflow and +# the ingest kept evicting each other; the group queues them now.) set -euo pipefail pip install --quiet --require-hashes --only-binary :all: -r requirements.txt python scripts/remove_games.py diff --git a/docs/ADMIN.md b/docs/ADMIN.md index 44460774..68867c79 100644 --- a/docs/ADMIN.md +++ b/docs/ADMIN.md @@ -659,10 +659,11 @@ counts as *no* — the page's *Last run* line says which one stopped it: - **The previous batch has been ingested**: `scripts/temp_info.jsonl` on `main` is empty. Checked again inside the commit itself, so a batch never lands in the middle of an ingest. -- **No data job is running or waiting.** The `data-write` group keeps one - waiting run, and the ingest a batch starts would cancel a job already waiting - its turn. The list of those workflows is `web/shared/pipeline.ts`, which a - test holds to the YAML. +- **No data job is running or waiting.** The `data-write` group queues its + runs (`queue: max`), so the ingest a batch starts would only wait behind such + a job; the gate makes a batch land when its ingest can start at once. The + list of those workflows is `web/shared/pipeline.ts`, which a test holds to + the YAML. - **Today's budget is not spent.** It is reserved before the commit and given back if the commit fails. diff --git a/web/admin/src/views/AutomationView.svelte b/web/admin/src/views/AutomationView.svelte index c68d0d66..fcf70bf4 100644 --- a/web/admin/src/views/AutomationView.svelte +++ b/web/admin/src/views/AutomationView.svelte @@ -194,7 +194,7 @@

A batch waits while the previous one is still in scripts/temp_info.jsonl, or while a data job is running or - waiting its turn - pushing then would cancel that job. + waiting its turn - the ingest it starts would only queue behind that job.

diff --git a/web/admin/src/views/PipelineView.svelte b/web/admin/src/views/PipelineView.svelte index 8451a508..121135c4 100644 --- a/web/admin/src/views/PipelineView.svelte +++ b/web/admin/src/views/PipelineView.svelte @@ -71,8 +71,8 @@

Pipeline

- The data workflows on GitHub Actions. Those marked data-write take turns: one runs, at most one waits, and a newer - one cancels the waiting run. + The data workflows on GitHub Actions. Those marked data-write take turns: one runs and the others wait in a queue, + first in, first out.