Every window swept before needs one more pass for the games without English, and backfill windows are dispatched several at a time. The discovery group kept ONE waiting run, so a third cancelled the one already waiting - the lesson data-write learned in October. - queue: max in both discovery workflows; pipeline.test.ts now holds every shared concurrency group to one queue setting. - Backfill Discovery takes max_pages (through env, default 40): a whole recent year is about 30 pages of 100. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
139 lines
5.5 KiB
YAML
139 lines
5.5 KiB
YAML
name: Backfill Discovery (TEMPORARY)
|
|
|
|
# >>> TEMPORARY. DELETE THIS FILE AND scripts/backfill_discover.py ONCE THE
|
|
# >>> HISTORICAL BACKLOG HAS BEEN WORKED THROUGH.
|
|
#
|
|
# Measured 2026-09-10: Steam listed 16,668 free games that support English;
|
|
# data/ held 3,424. This workflow chews through that gap one release window at
|
|
# a time. `Discover New Games` covers everything from today forward and is
|
|
# permanent -- this one is not.
|
|
#
|
|
# Since v4.6.0 discovery takes games in every language (18,790 free games on
|
|
# 2026-10-30, 1,592 of them without English), so a window swept before then
|
|
# needs one more pass. A whole recent year is ~30 search pages; a window is
|
|
# done only when the log says "walked past the window".
|
|
#
|
|
# Same propose-only contract as the daily job: `contents: read`, and the same
|
|
# guard step asserting no dataset-writing helper is reachable.
|
|
|
|
on:
|
|
schedule:
|
|
# Sundays 17:30 UTC -- a different day AND a different hour from the daily
|
|
# discovery run (05:00), because both draw on the same per-IP Steam budget.
|
|
- cron: '30 17 * * 0'
|
|
workflow_dispatch:
|
|
inputs:
|
|
year:
|
|
description: 'Release year to sweep'
|
|
required: true
|
|
quarter:
|
|
description: 'Quarter (1-4). Leave blank to sweep the whole year.'
|
|
required: false
|
|
default: ''
|
|
limit:
|
|
description: 'Max candidates to propose in one run'
|
|
required: false
|
|
default: '250'
|
|
max_pages:
|
|
description: 'Search pages to walk once the window is found (100 rows each; a whole recent year is ~30)'
|
|
required: false
|
|
default: '40'
|
|
dry_run:
|
|
description: 'Find candidates but do not POST them'
|
|
type: boolean
|
|
required: false
|
|
default: false
|
|
|
|
# The window the SCHEDULED run sweeps. Dispatch runs use their own inputs.
|
|
#
|
|
# Bump this as windows complete: the job prints "candidates: N" at the end, and
|
|
# a window is done when a re-run reports 0. Working newest-first keeps the most
|
|
# relevant games arriving first. When you reach the end of the backlog, delete
|
|
# this workflow rather than leaving it grinding over swept ground.
|
|
env:
|
|
SWEEP_YEAR: '2026'
|
|
SWEEP_QUARTER: '2'
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
# Shares the `discovery` group with the daily job on purpose: two sweeps
|
|
# hammering Steam's search from the same runner pool is the one way to earn a
|
|
# rate limit that hurts both. `queue: max`, like the daily job: windows are
|
|
# dispatched several at a time, and the default single waiting slot would
|
|
# cancel all but the last.
|
|
group: discovery
|
|
cancel-in-progress: false
|
|
queue: max
|
|
|
|
jobs:
|
|
backfill:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 90
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
|
|
- uses: actions/setup-python@v6
|
|
with:
|
|
python-version: '3.15'
|
|
|
|
- name: Assert discovery cannot write the dataset
|
|
run: |
|
|
if grep -nE '\b(save_main|save_jsonl|clear_temp|fetch_full)\b' \
|
|
scripts/discover_new.py scripts/backfill_discover.py; then
|
|
echo "::error::A discovery script imports a dataset-writing helper."
|
|
echo "Discovery must only PROPOSE candidates to the review queue."
|
|
exit 1
|
|
fi
|
|
echo "OK: no dataset-writing helpers referenced."
|
|
|
|
- name: Check ingest credentials
|
|
env:
|
|
WORKER_BASE_URL: ${{ secrets.WORKER_BASE_URL }}
|
|
CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
|
|
CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
|
|
run: |
|
|
missing=0
|
|
for v in WORKER_BASE_URL CF_ACCESS_CLIENT_ID CF_ACCESS_CLIENT_SECRET; do
|
|
if [ -z "${!v}" ]; then echo "::error::missing secret $v"; missing=1; fi
|
|
done
|
|
if [ "$missing" = "1" ]; then
|
|
echo "See docs/ADMIN.md for the Access service token setup."
|
|
exit 1
|
|
fi
|
|
|
|
- run: |
|
|
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
|
|
|
|
- name: Backfill
|
|
env:
|
|
WORKER_BASE_URL: ${{ secrets.WORKER_BASE_URL }}
|
|
CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
|
|
CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
|
|
YEAR: ${{ inputs.year }}
|
|
QUARTER: ${{ inputs.quarter }}
|
|
LIMIT: ${{ inputs.limit || '250' }}
|
|
MAX_PAGES: ${{ inputs.max_pages || '40' }}
|
|
DRY: ${{ inputs.dry_run && '--dry-run' || '' }}
|
|
run: |
|
|
# An absent YEAR means this is the scheduled run, so both env defaults
|
|
# apply. Resolved in shell, NOT with an expression fallback in the env
|
|
# block above: `year` is required on dispatch but `quarter` is
|
|
# deliberately blankable, and a fallback cannot tell "blank, meaning
|
|
# whole year" from "absent, meaning use the default" -- it would
|
|
# silently narrow an explicit whole-year request to SWEEP_QUARTER.
|
|
#
|
|
# (Actions also evaluates expression syntax inside `run:` comments, so
|
|
# writing the rejected form out here would itself break the file.)
|
|
if [ -z "$YEAR" ]; then
|
|
YEAR="$SWEEP_YEAR"
|
|
QUARTER="$SWEEP_QUARTER"
|
|
fi
|
|
# A blank quarter must be omitted from argv, not passed as an empty
|
|
# string: argparse rejects '' as an invalid choice.
|
|
args="--year $YEAR --limit $LIMIT --max-pages $MAX_PAGES"
|
|
if [ -n "$QUARTER" ]; then args="$args --quarter $QUARTER"; fi
|
|
echo "backfill_discover.py $args $DRY"
|
|
python scripts/backfill_discover.py $args $DRY
|