ci(discovery): queue waiting runs and sweep a whole year

Every window swept before needs one more pass for the games without
English, and backfill windows are dispatched several at a time. The
discovery group kept ONE waiting run, so a third cancelled the one
already waiting - the lesson data-write learned in October.

- queue: max in both discovery workflows; pipeline.test.ts now holds
  every shared concurrency group to one queue setting.
- Backfill Discovery takes max_pages (through env, default 40): a whole
  recent year is about 30 pages of 100.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
poli0981andClaude Opus 5.5 committed 2026-10-30 17:28:27 +07:00
1 parent 6f6eb3a71f
commit 4672f871b3
3 files changed
+42 -7

No files matched your search

+19 -6
View File
@@ -3,10 +3,15 @@ name: Backfill Discovery (TEMPORARY)
# >>> TEMPORARY. DELETE THIS FILE AND scripts/backfill_discover.py ONCE THE
# >>> HISTORICAL BACKLOG HAS BEEN WORKED THROUGH.
#
# Measured 2026-09-10: Steam lists 16,668 free games matching the catalogue's
# own filters; data/ holds 3,424. This workflow chews through that gap one
# release window at a time. `Discover New Games` covers everything from today
# forward and is permanent -- this one is not.
# Measured 2026-09-10: Steam listed 16,668 free games that support English;
# data/ held 3,424. This workflow chews through that gap one release window at
# a time. `Discover New Games` covers everything from today forward and is
# permanent -- this one is not.
#
# Since v4.6.0 discovery takes games in every language (18,790 free games on
# 2026-10-30, 1,592 of them without English), so a window swept before then
# needs one more pass. A whole recent year is ~30 search pages; a window is
# done only when the log says "walked past the window".
#
# Same propose-only contract as the daily job: `contents: read`, and the same
# guard step asserting no dataset-writing helper is reachable.
@@ -29,6 +34,10 @@ on:
description: 'Max candidates to propose in one run'
required: false
default: '250'
max_pages:
description: 'Search pages to walk once the window is found (100 rows each; a whole recent year is ~30)'
required: false
default: '40'
dry_run:
description: 'Find candidates but do not POST them'
type: boolean
@@ -51,9 +60,12 @@ permissions:
concurrency:
# Shares the `discovery` group with the daily job on purpose: two sweeps
# hammering Steam's search from the same runner pool is the one way to earn a
# rate limit that hurts both.
# rate limit that hurts both. `queue: max`, like the daily job: windows are
# dispatched several at a time, and the default single waiting slot would
# cancel all but the last.
group: discovery
cancel-in-progress: false
queue: max
jobs:
backfill:
@@ -102,6 +114,7 @@ jobs:
YEAR: ${{ inputs.year }}
QUARTER: ${{ inputs.quarter }}
LIMIT: ${{ inputs.limit || '250' }}
MAX_PAGES: ${{ inputs.max_pages || '40' }}
DRY: ${{ inputs.dry_run && '--dry-run' || '' }}
run: |
# An absent YEAR means this is the scheduled run, so both env defaults
@@ -119,7 +132,7 @@ jobs:
fi
# A blank quarter must be omitted from argv, not passed as an empty
# string: argparse rejects '' as an invalid choice.
args="--year $YEAR --limit $LIMIT"
args="--year $YEAR --limit $LIMIT --max-pages $MAX_PAGES"
if [ -n "$QUARTER" ]; then args="$args --quarter $QUARTER"; fi
echo "backfill_discover.py $args $DRY"
python scripts/backfill_discover.py $args $DRY
+7 -1
View File
@@ -34,11 +34,17 @@ permissions:
concurrency:
# Its own group, NOT `data-write`: this job writes nothing to the repo, so
# queueing it behind the ~115-minute data jobs would delay discovery for no
# queueing it behind the long data jobs would delay discovery for no
# benefit. Two overlapping discovery runs are the only real hazard, and this
# covers that.
#
# `queue: max` keeps every waiting run (up to 100, first in, first out).
# Without it the group keeps ONE: a third run - a backfill dispatched while
# another waits - cancels the one already waiting. Every member of the group
# says the same (Backfill Discovery too); web/shared/pipeline.test.ts holds it.
group: discovery
cancel-in-progress: false
queue: max
jobs:
discover:
+16
View File
@@ -13,6 +13,9 @@
* rebase conflict in data/index.json;
* - declare `timeout-minutes`, and a JOB_BUDGET_MINUTES at least 20 below
* it (scripts/core/budget.py): the script stops in time to save and push.
*
* The same holds for any shared group (`discovery`): its members queue, or
* none of them does.
*/
import { describe, expect, it } from "vitest";
import { existsSync, readdirSync, readFileSync } from "node:fs";
@@ -78,6 +81,19 @@ describe("every data-write workflow", () => {
}
});
it("shares its queue setting with every workflow in its group", () => {
// A group named by an expression is one run's own; only fixed names are shared.
const byGroup = new Map<string, boolean[]>();
for (const f of readdirSync(DIR).filter((name) => name.endsWith(".yml"))) {
const text = code(`.github/workflows/${f}`);
const group = /^\s*group\s*:\s*([\w-]+)\s*$/m.exec(text)?.[1];
if (!group) continue;
byGroup.set(group, [...(byGroup.get(group) ?? []), /\bqueue\s*:\s*max\b/.test(text)]);
}
expect(byGroup.get("discovery")).toEqual([true, true]);
for (const [group, queues] of byGroup) expect(new Set(queues).size, group).toBe(1);
});
it("stops at its budget in time to save before its timeout", () => {
for (const file of dataWrite) {
const text = code(file);