ci(pipeline): queue data-write runs and start each on the latest main
- queue: max on every data-write workflow (GitHub, 2026-05-07): up to 100 runs wait, first in, first out. The default kept one, and a run queueing behind a running job cancelled it. - ref: main on every checkout. A run's commit is fixed when it is created, so a run that waited behind another data job wrote shards from before that job's commit; Auto Update JSON failed four days in ten on the rebase conflict in data/index.json. - timeout-minutes everywhere, and JOB_BUDGET_MINUTES at least 20 below it for the long loops (never for the ingest, which empties its queue). - pipeline.test.ts fails on a data-write workflow missing any of these. - The admin's Pipeline and Automation pages, docs/ADMIN.md and the auto-approve comment no longer say a push evicts a waiting job; the idle gate stays, so a batch's ingest starts at once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6e7d1bcf15
commit
529b8603c3
12 files changed
+146
-29
No files matched your search
@@ -5,26 +5,31 @@ on :
|
||||
# fires the ingest on a detached HEAD, and dies with exit 128 on push.
|
||||
#
|
||||
# data/blocklist.jsonl is here too: this job also applies removals
|
||||
# (bash/ingest.sh runs scripts/remove_games.py first). A separate removal
|
||||
# workflow would share `data-write` with this one, and each would evict the
|
||||
# other's pending run.
|
||||
# (bash/ingest.sh runs scripts/remove_games.py first), so one run does both.
|
||||
push : { branches : [ main ], paths : [ 'scripts/temp_info.jsonl', 'data/blocklist.jsonl' ] }
|
||||
# Orphan recovery, not a poll. `data-write` keeps at most ONE run pending and
|
||||
# cancels the older one, so a push here that queues behind a ~115-minute data
|
||||
# job is silently evicted the moment any OTHER data-write workflow queues --
|
||||
# and nothing retries it, leaving approved games sitting in temp_info.jsonl
|
||||
# until the next unrelated push. Re-running on a clear file is free:
|
||||
# ingest_new.py returns immediately and `git diff --staged --quiet` skips the
|
||||
# commit, so this is idempotent by construction.
|
||||
# A safety net, not a poll. Before data-write queued its runs (queue: max),
|
||||
# a push here that waited behind a long data job was evicted the moment any
|
||||
# other data-write workflow queued, and approved games sat in
|
||||
# temp_info.jsonl until the next unrelated push. Nothing is evicted now, but
|
||||
# a failed run still leaves its links queued. Re-running on a clear file is
|
||||
# free: ingest_new.py returns immediately and `git diff --staged --quiet`
|
||||
# skips the commit, so this is idempotent by construction.
|
||||
schedule : [ { cron : '0 */3 * * *' } ]
|
||||
workflow_dispatch :
|
||||
permissions : { contents : write }
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency :
|
||||
group : data-write
|
||||
cancel-in-progress : false
|
||||
queue : max
|
||||
jobs :
|
||||
ingest :
|
||||
runs-on : ubuntu-latest
|
||||
# About seven seconds a game, and an /admin batch holds at most 100. No
|
||||
# time budget: ingest_new.py empties the whole queue file when it is done,
|
||||
# so a game it had not reached would be lost.
|
||||
timeout-minutes : 120
|
||||
steps :
|
||||
-
|
||||
uses : actions/checkout@v6
|
||||
|
||||
@@ -19,13 +19,17 @@ permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
jobs:
|
||||
mark:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# The tip of main when the job starts, not when it was queued: a run
|
||||
|
||||
@@ -4,15 +4,22 @@ on:
|
||||
workflow_dispatch:
|
||||
permissions: {contents: write}
|
||||
# Shares the data-write group with every other workflow that commits to
|
||||
# data/ — without it this can push mid-run against a 2-hour job.
|
||||
# data/ — without it this can push mid-run against a long job. queue: max -
|
||||
# a run that finds another data job running waits its turn (first in, first
|
||||
# out) instead of evicting the run already waiting.
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
jobs:
|
||||
snapshot:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
# main as it is when the job STARTS, so the snapshot holds what the
|
||||
# jobs before it committed.
|
||||
- uses: actions/checkout@v4
|
||||
with: {ref: main}
|
||||
- uses: actions/setup-python@v5
|
||||
with: {python-version: '3.15'}
|
||||
- run: chmod +x ./bash/snapshot.sh && ./bash/snapshot.sh
|
||||
@@ -5,15 +5,27 @@ on:
|
||||
schedule: [{cron: '0 5 1,15 * *'}]
|
||||
workflow_dispatch:
|
||||
permissions: {contents: write}
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
jobs:
|
||||
leaderboard:
|
||||
runs-on: ubuntu-latest
|
||||
# About 0.55 s a game through the keyed Web API: ~65 minutes for 7,000
|
||||
# offline games, ~2 hours at 14,000. The budget stops the sampling with
|
||||
# time left to save and commit.
|
||||
timeout-minutes: 240
|
||||
steps:
|
||||
# main as it is when the job STARTS: a run that waited its turn must
|
||||
# work on what the job before it committed.
|
||||
- uses: actions/checkout@v4
|
||||
with: {ref: main}
|
||||
- uses: actions/setup-python@v5
|
||||
with: {python-version: '3.15'}
|
||||
- run: chmod +x ./bash/offline.sh && ./bash/offline.sh
|
||||
env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'}
|
||||
env:
|
||||
STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'
|
||||
JOB_BUDGET_MINUTES: '200'
|
||||
@@ -3,15 +3,27 @@ on:
|
||||
schedule: [{cron: '0 3 * * 0,3,5'}]
|
||||
workflow_dispatch:
|
||||
permissions: {contents: write}
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
jobs:
|
||||
leaderboard:
|
||||
runs-on: ubuntu-latest
|
||||
# The player counts come from the keyed Web API with no pause between
|
||||
# batches, about 0.55 s a game: ~27 minutes for 2,900 online games. The
|
||||
# budget stops the sampling with time left to save and commit.
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
# main as it is when the job STARTS: a run that waited its turn must
|
||||
# work on what the job before it committed.
|
||||
- uses: actions/checkout@v4
|
||||
with: {ref: main}
|
||||
- uses: actions/setup-python@v5
|
||||
with: {python-version: '3.15'}
|
||||
- run: chmod +x ./bash/online.sh && ./bash/online.sh
|
||||
env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'}
|
||||
env:
|
||||
STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'
|
||||
JOB_BUDGET_MINUTES: '120'
|
||||
@@ -3,15 +3,26 @@ on:
|
||||
schedule: [{cron: '0 0 * * *'}]
|
||||
workflow_dispatch:
|
||||
permissions: {contents: write}
|
||||
# queue: max - a run that finds another data job running waits its turn
|
||||
# (first in, first out) instead of evicting the run already waiting.
|
||||
concurrency:
|
||||
group: data-write
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
jobs:
|
||||
update:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
# main as it is when the job STARTS. The commit the scheduled run was
|
||||
# created on is older whenever another data job ran first, and this one
|
||||
# then failed on a conflict in data/index.json (four times in ten days,
|
||||
# October 2026).
|
||||
- uses: actions/checkout@v6
|
||||
with: {ref: main}
|
||||
- uses: actions/setup-python@v6
|
||||
with: {python-version: '3.15'}
|
||||
- run: chmod +x ./bash/json.sh && ./bash/json.sh
|
||||
env: {STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'}
|
||||
env:
|
||||
STEAM_API_KEY: '${{ secrets.STEAM_API_KEY }}'
|
||||
JOB_BUDGET_MINUTES: '60'
|
||||
+4
-3
@@ -3,9 +3,10 @@
|
||||
#
|
||||
# remove_games.py applies data/blocklist.jsonl (games the maintainer removed in
|
||||
# /admin or with delete_game.py) before ingest_new.py adds the queued links.
|
||||
# One workflow does both on purpose: `data-write` keeps ONE pending run and a
|
||||
# newer one evicts it, so a separate removal workflow and the ingest would keep
|
||||
# cancelling each other. Whichever run survives does both.
|
||||
# One workflow does both: a commit to either file starts one run that settles
|
||||
# the two, in that order. (It was also the only way to do it while
|
||||
# `data-write` kept a single pending run and a separate removal workflow and
|
||||
# the ingest kept evicting each other; the group queues them now.)
|
||||
set -euo pipefail
|
||||
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
|
||||
python scripts/remove_games.py
|
||||
|
||||
+5
-4
@@ -659,10 +659,11 @@ counts as *no* — the page's *Last run* line says which one stopped it:
|
||||
- **The previous batch has been ingested**: `scripts/temp_info.jsonl` on `main`
|
||||
is empty. Checked again inside the commit itself, so a batch never lands in
|
||||
the middle of an ingest.
|
||||
- **No data job is running or waiting.** The `data-write` group keeps one
|
||||
waiting run, and the ingest a batch starts would cancel a job already waiting
|
||||
its turn. The list of those workflows is `web/shared/pipeline.ts`, which a
|
||||
test holds to the YAML.
|
||||
- **No data job is running or waiting.** The `data-write` group queues its
|
||||
runs (`queue: max`), so the ingest a batch starts would only wait behind such
|
||||
a job; the gate makes a batch land when its ingest can start at once. The
|
||||
list of those workflows is `web/shared/pipeline.ts`, which a test holds to
|
||||
the YAML.
|
||||
- **Today's budget is not spent.** It is reserved before the commit and given
|
||||
back if the commit fails.
|
||||
|
||||
|
||||
@@ -194,7 +194,7 @@
|
||||
</Button>
|
||||
<p class="text-xs text-muted-foreground">
|
||||
A batch waits while the previous one is still in scripts/temp_info.jsonl, or while a data job is running or
|
||||
waiting its turn - pushing then would cancel that job.
|
||||
waiting its turn - the ingest it starts would only queue behind that job.
|
||||
</p>
|
||||
</article>
|
||||
|
||||
|
||||
@@ -71,8 +71,8 @@
|
||||
<div>
|
||||
<h1 class="text-2xl font-semibold tracking-tight">Pipeline</h1>
|
||||
<p class="max-w-3xl text-sm text-muted-foreground">
|
||||
The data workflows on GitHub Actions. Those marked data-write take turns: one runs, at most one waits, and a newer
|
||||
one cancels the waiting run.
|
||||
The data workflows on GitHub Actions. Those marked data-write take turns: one runs and the others wait in a queue,
|
||||
first in, first out.
|
||||
</p>
|
||||
</div>
|
||||
<Button variant="outline" size="sm" onclick={() => load(true)} disabled={loading}>
|
||||
|
||||
@@ -1,7 +1,18 @@
|
||||
/**
|
||||
* DATA_WORKFLOWS must name every workflow in `concurrency: data-write`:
|
||||
* auto-approve waits for exactly those, and one missing from the list is one
|
||||
* whose pending run an automatic batch could evict.
|
||||
* auto-approve waits for exactly those.
|
||||
*
|
||||
* And every one of them must:
|
||||
* - queue (`queue: max`, `cancel-in-progress: false`). With the default a
|
||||
* run that queued behind a running job evicted the one already waiting:
|
||||
* Update Reviews lost five of ten runs that way in October 2026. Mixing
|
||||
* the two in one group is not documented, so none may;
|
||||
* - check out `main`. A run's commit is fixed when the run is created, and a
|
||||
* run that waited behind another data job would write shards from before
|
||||
* that job's commit - Auto Update JSON failed four days in ten on the
|
||||
* rebase conflict in data/index.json;
|
||||
* - declare `timeout-minutes`, and a JOB_BUDGET_MINUTES at least 20 below
|
||||
* it (scripts/core/budget.py): the script stops in time to save and push.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||||
@@ -11,6 +22,26 @@ import { DATA_WORKFLOWS } from "./pipeline";
|
||||
const REPO = join(__dirname, "..", "..");
|
||||
const DIR = join(REPO, ".github", "workflows");
|
||||
|
||||
/** The YAML without its comment lines, which quote the old settings. */
|
||||
function code(file: string): string {
|
||||
return readFileSync(join(REPO, file), "utf-8")
|
||||
.split("\n")
|
||||
.filter((line) => !/^\s*#/.test(line))
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
const TIMEOUT = /timeout-minutes\s*:\s*(.+)/;
|
||||
const BUDGET = /JOB_BUDGET_MINUTES\s*:\s*(.+)/;
|
||||
|
||||
/** The whole numbers on the first line `key` matches: [60] for "60", [340, 45]
|
||||
* for "${{ inputs.backfill && 340 || 45 }}". */
|
||||
function numbers(text: string, key: RegExp): number[] {
|
||||
const line = key.exec(text);
|
||||
return line ? (line[1].match(/\d+/g) ?? []).map(Number) : [];
|
||||
}
|
||||
|
||||
const dataWrite = DATA_WORKFLOWS.filter((w) => w.dataWrite).map((w) => w.file);
|
||||
|
||||
describe("DATA_WORKFLOWS", () => {
|
||||
it("marks exactly the data-write workflows", () => {
|
||||
const inGroup = readdirSync(DIR)
|
||||
@@ -27,3 +58,35 @@ describe("DATA_WORKFLOWS", () => {
|
||||
expect(new Set(DATA_WORKFLOWS.map((w) => w.file)).size).toBe(DATA_WORKFLOWS.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe("every data-write workflow", () => {
|
||||
it("queues its runs instead of evicting the one waiting", () => {
|
||||
for (const file of dataWrite) {
|
||||
const text = code(file);
|
||||
expect(text, file).toMatch(/\bqueue\s*:\s*max\b/);
|
||||
expect(text, file).toMatch(/\bcancel-in-progress\s*:\s*false\b/);
|
||||
}
|
||||
});
|
||||
|
||||
it("checks out main, not the commit its run was created on", () => {
|
||||
for (const file of dataWrite) {
|
||||
const text = code(file);
|
||||
const checkouts = text.match(/uses\s*:\s*actions\/checkout@/g) ?? [];
|
||||
const refs = text.match(/\bref\s*:\s*main\b/g) ?? [];
|
||||
expect(checkouts.length, file).toBeGreaterThan(0);
|
||||
expect(refs.length, file).toBe(checkouts.length);
|
||||
}
|
||||
});
|
||||
|
||||
it("stops at its budget in time to save before its timeout", () => {
|
||||
for (const file of dataWrite) {
|
||||
const text = code(file);
|
||||
const timeouts = numbers(text, TIMEOUT);
|
||||
const budgets = numbers(text, BUDGET);
|
||||
expect(timeouts.length, file).toBeGreaterThan(0);
|
||||
if (budgets.length === 0) continue;
|
||||
expect(budgets.length, file).toBe(timeouts.length);
|
||||
budgets.forEach((b, i) => expect(b + 20, `${file}: budget ${b}, timeout ${timeouts[i]}`).toBeLessThanOrEqual(timeouts[i]));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -17,9 +17,10 @@
|
||||
* ingested. Checked again atomically inside the commit
|
||||
* (requireEmptyQueue). Not "no row is approved": a network-skipped row
|
||||
* stays approved for 12 hours and would stall everything that long;
|
||||
* - no data-write workflow is running or waiting (GitHub Actions). That
|
||||
* group keeps ONE pending run, and the ingest this commit starts would
|
||||
* evict a job already waiting its turn;
|
||||
* - no data-write workflow is running or waiting (GitHub Actions). The
|
||||
* group queues its runs (queue: max), so the ingest this commit starts
|
||||
* would no longer evict a job waiting its turn - it would wait behind it.
|
||||
* The gate stays so that a batch lands when the ingest can start at once;
|
||||
* - something is eligible: pending, free, app_type game, health ok, old
|
||||
* enough, from an allowed source, never decided, its game not committed,
|
||||
* not in data/blocklist.jsonl.
|
||||
|
||||
Reference in new issue
Block a user