ci(discovery): queue waiting runs and sweep a whole year
Every window swept before needs one more pass for the games without English, and backfill windows are dispatched several at a time. The discovery group kept ONE waiting run, so a third cancelled the one already waiting - the lesson data-write learned in October. - queue: max in both discovery workflows; pipeline.test.ts now holds every shared concurrency group to one queue setting. - Backfill Discovery takes max_pages (through env, default 40): a whole recent year is about 30 pages of 100. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6f6eb3a71f
commit
4672f871b3
3 files changed
+42
-7
No files matched your search
@@ -3,10 +3,15 @@ name: Backfill Discovery (TEMPORARY)
|
||||
# >>> TEMPORARY. DELETE THIS FILE AND scripts/backfill_discover.py ONCE THE
|
||||
# >>> HISTORICAL BACKLOG HAS BEEN WORKED THROUGH.
|
||||
#
|
||||
# Measured 2026-09-10: Steam lists 16,668 free games matching the catalogue's
|
||||
# own filters; data/ holds 3,424. This workflow chews through that gap one
|
||||
# release window at a time. `Discover New Games` covers everything from today
|
||||
# forward and is permanent -- this one is not.
|
||||
# Measured 2026-09-10: Steam listed 16,668 free games that support English;
|
||||
# data/ held 3,424. This workflow chews through that gap one release window at
|
||||
# a time. `Discover New Games` covers everything from today forward and is
|
||||
# permanent -- this one is not.
|
||||
#
|
||||
# Since v4.6.0 discovery takes games in every language (18,790 free games on
|
||||
# 2026-10-30, 1,592 of them without English), so a window swept before then
|
||||
# needs one more pass. A whole recent year is ~30 search pages; a window is
|
||||
# done only when the log says "walked past the window".
|
||||
#
|
||||
# Same propose-only contract as the daily job: `contents: read`, and the same
|
||||
# guard step asserting no dataset-writing helper is reachable.
|
||||
@@ -29,6 +34,10 @@ on:
|
||||
description: 'Max candidates to propose in one run'
|
||||
required: false
|
||||
default: '250'
|
||||
max_pages:
|
||||
description: 'Search pages to walk once the window is found (100 rows each; a whole recent year is ~30)'
|
||||
required: false
|
||||
default: '40'
|
||||
dry_run:
|
||||
description: 'Find candidates but do not POST them'
|
||||
type: boolean
|
||||
@@ -51,9 +60,12 @@ permissions:
|
||||
concurrency:
|
||||
# Shares the `discovery` group with the daily job on purpose: two sweeps
|
||||
# hammering Steam's search from the same runner pool is the one way to earn a
|
||||
# rate limit that hurts both.
|
||||
# rate limit that hurts both. `queue: max`, like the daily job: windows are
|
||||
# dispatched several at a time, and the default single waiting slot would
|
||||
# cancel all but the last.
|
||||
group: discovery
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
jobs:
|
||||
backfill:
|
||||
@@ -102,6 +114,7 @@ jobs:
|
||||
YEAR: ${{ inputs.year }}
|
||||
QUARTER: ${{ inputs.quarter }}
|
||||
LIMIT: ${{ inputs.limit || '250' }}
|
||||
MAX_PAGES: ${{ inputs.max_pages || '40' }}
|
||||
DRY: ${{ inputs.dry_run && '--dry-run' || '' }}
|
||||
run: |
|
||||
# An absent YEAR means this is the scheduled run, so both env defaults
|
||||
@@ -119,7 +132,7 @@ jobs:
|
||||
fi
|
||||
# A blank quarter must be omitted from argv, not passed as an empty
|
||||
# string: argparse rejects '' as an invalid choice.
|
||||
args="--year $YEAR --limit $LIMIT"
|
||||
args="--year $YEAR --limit $LIMIT --max-pages $MAX_PAGES"
|
||||
if [ -n "$QUARTER" ]; then args="$args --quarter $QUARTER"; fi
|
||||
echo "backfill_discover.py $args $DRY"
|
||||
python scripts/backfill_discover.py $args $DRY
|
||||
@@ -34,11 +34,17 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
# Its own group, NOT `data-write`: this job writes nothing to the repo, so
|
||||
# queueing it behind the ~115-minute data jobs would delay discovery for no
|
||||
# queueing it behind the long data jobs would delay discovery for no
|
||||
# benefit. Two overlapping discovery runs are the only real hazard, and this
|
||||
# covers that.
|
||||
#
|
||||
# `queue: max` keeps every waiting run (up to 100, first in, first out).
|
||||
# Without it the group keeps ONE: a third run - a backfill dispatched while
|
||||
# another waits - cancels the one already waiting. Every member of the group
|
||||
# says the same (Backfill Discovery too); web/shared/pipeline.test.ts holds it.
|
||||
group: discovery
|
||||
cancel-in-progress: false
|
||||
queue: max
|
||||
|
||||
jobs:
|
||||
discover:
|
||||
|
||||
@@ -13,6 +13,9 @@
|
||||
* rebase conflict in data/index.json;
|
||||
* - declare `timeout-minutes`, and a JOB_BUDGET_MINUTES at least 20 below
|
||||
* it (scripts/core/budget.py): the script stops in time to save and push.
|
||||
*
|
||||
* The same holds for any shared group (`discovery`): its members queue, or
|
||||
* none of them does.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||||
@@ -78,6 +81,19 @@ describe("every data-write workflow", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("shares its queue setting with every workflow in its group", () => {
|
||||
// A group named by an expression is one run's own; only fixed names are shared.
|
||||
const byGroup = new Map<string, boolean[]>();
|
||||
for (const f of readdirSync(DIR).filter((name) => name.endsWith(".yml"))) {
|
||||
const text = code(`.github/workflows/${f}`);
|
||||
const group = /^\s*group\s*:\s*([\w-]+)\s*$/m.exec(text)?.[1];
|
||||
if (!group) continue;
|
||||
byGroup.set(group, [...(byGroup.get(group) ?? []), /\bqueue\s*:\s*max\b/.test(text)]);
|
||||
}
|
||||
expect(byGroup.get("discovery")).toEqual([true, true]);
|
||||
for (const [group, queues] of byGroup) expect(new Set(queues).size, group).toBe(1);
|
||||
});
|
||||
|
||||
it("stops at its budget in time to save before its timeout", () => {
|
||||
for (const file of dataWrite) {
|
||||
const text = code(file);
|
||||
|
||||
Reference in new issue
Block a user