Every window swept before needs one more pass for the games without English, and backfill windows are dispatched several at a time. The discovery group kept ONE waiting run, so a third cancelled the one already waiting - the lesson data-write learned in October. - queue: max in both discovery workflows; pipeline.test.ts now holds every shared concurrency group to one queue setting. - Backfill Discovery takes max_pages (through env, default 40): a whole recent year is about 30 pages of 100. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
103 lines
4.1 KiB
YAML
103 lines
4.1 KiB
YAML
name: Discover New Games
|
|
|
|
# Finds newly-released free-to-play titles on Steam and PROPOSES them to the
|
|
# admin review queue in D1. It cannot add a game to the catalogue: publication
|
|
# is decided under /admin - by a reviewer, or by the auto-approve rule a
|
|
# reviewer turned on there (off by default, capped per day) - and is still
|
|
# only a request that ingest_new.py checks game by game.
|
|
#
|
|
# The `permissions: contents: read` below is the enforcement, not a formality.
|
|
# Every other data workflow in this repo declares `contents: write`; this one
|
|
# must not, because a discovery job that could also commit would collapse the
|
|
# propose/approve split that the whole ingest design rests on.
|
|
|
|
on:
|
|
schedule:
|
|
# 05:00 UTC. Deliberately clear of `Auto Update JSON Data` (00:00) and of
|
|
# Backfill Discovery (Sundays 17:30) so the three never contend for the
|
|
# Steam rate budget, which is per-IP and therefore shared across runners.
|
|
- cron: '0 5 * * *'
|
|
workflow_dispatch:
|
|
inputs:
|
|
max_pages:
|
|
description: 'Search pages to walk (100 rows each). Use ~10 for the initial catch-up sweep.'
|
|
required: false
|
|
default: '5'
|
|
dry_run:
|
|
description: 'Find candidates but do not POST them'
|
|
type: boolean
|
|
required: false
|
|
default: false
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
# Its own group, NOT `data-write`: this job writes nothing to the repo, so
|
|
# queueing it behind the long data jobs would delay discovery for no
|
|
# benefit. Two overlapping discovery runs are the only real hazard, and this
|
|
# covers that.
|
|
#
|
|
# `queue: max` keeps every waiting run (up to 100, first in, first out).
|
|
# Without it the group keeps ONE: a third run - a backfill dispatched while
|
|
# another waits - cancels the one already waiting. Every member of the group
|
|
# says the same (Backfill Discovery too); web/shared/pipeline.test.ts holds it.
|
|
group: discovery
|
|
cancel-in-progress: false
|
|
queue: max
|
|
|
|
jobs:
|
|
discover:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 60
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
|
|
- uses: actions/setup-python@v6
|
|
with:
|
|
python-version: '3.15'
|
|
|
|
# Belt-and-braces for the propose-only guarantee above. `contents: read`
|
|
# already blocks a push, but this fails the job at the point the intent is
|
|
# violated rather than at the point the token is refused -- and it names
|
|
# the reason, which a 403 from git does not.
|
|
- name: Assert discovery cannot write the dataset
|
|
run: |
|
|
if grep -nE '\b(save_main|save_jsonl|clear_temp|fetch_full)\b' \
|
|
scripts/discover_new.py scripts/backfill_discover.py; then
|
|
echo "::error::A discovery script imports a dataset-writing helper."
|
|
echo "Discovery must only PROPOSE candidates to the review queue."
|
|
exit 1
|
|
fi
|
|
echo "OK: no dataset-writing helpers referenced."
|
|
|
|
- name: Check ingest credentials
|
|
env:
|
|
WORKER_BASE_URL: ${{ secrets.WORKER_BASE_URL }}
|
|
CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
|
|
CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
|
|
run: |
|
|
missing=0
|
|
for v in WORKER_BASE_URL CF_ACCESS_CLIENT_ID CF_ACCESS_CLIENT_SECRET; do
|
|
if [ -z "${!v}" ]; then echo "::error::missing secret $v"; missing=1; fi
|
|
done
|
|
if [ "$missing" = "1" ]; then
|
|
echo "Create a Cloudflare Access service token named 'f2p-discovery'"
|
|
echo "and an Access application covering /api/ingest/*, then add the"
|
|
echo "client id/secret as repository secrets. See docs/ADMIN.md."
|
|
exit 1
|
|
fi
|
|
|
|
- run: |
|
|
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
|
|
|
|
- name: Discover
|
|
env:
|
|
WORKER_BASE_URL: ${{ secrets.WORKER_BASE_URL }}
|
|
CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
|
|
CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
|
|
run: |
|
|
python scripts/discover_new.py \
|
|
--max-pages "${{ inputs.max_pages || '5' }}" \
|
|
${{ inputs.dry_run && '--dry-run' || '' }}
|