The first SonarQube Cloud scan reported 57 issues. 45 are fixed in code, the 11 on the saved itch.io test pages leave the scan via .sonarcloud.properties, and css:S8776 (Tailwind's @utility nesting) is a false positive to mark on SonarQube Cloud. Worker: trailing slashes and the gate cookie's padding are trimmed by trimEnd() instead of /\/+$/ (quadratic on the public 404 path); the feed and collection parsers lose their backtracking regexes; compareCodeUnits keeps the code-unit sorts (the rule's localeCompare is locale-dependent); Number.*, DEFAULT_SIZES, crypto.randomUUID; i18n-edit.mjs reads stdin. Pipeline: the NSFW notices are one selector list, either class (the rule's ".a.b" would need both), each tested; plain if-guards; update_info.main returns None; no plain-http adapter; [[ ]] in cf_build_status.sh. Supply chain: requirements*.txt are hash-locked pip-tools compiles of requirements*.in (on Python 3.14, which Dependabot reads from the header), installed with --require-hashes --only-binary :all: in block scalars; npm ci --ignore-scripts in CI and the release builds (verified by manual release runs on all platforms); npm run cf:check. Release 4.4.2: also ships the toolchain upgrade (#149, #154) and Dependabot #141-#148, #150, #152, #153. CLAUDE.md gotchas 40 and 41. [skip-discuss] Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
164 lines
6.1 KiB
YAML
164 lines
6.1 KiB
YAML
# Daily rotating catalog health check (replaces check_paid / check_alive /
|
|
# update_reviews / update_status, which each crawled every page and timed out).
|
|
#
|
|
# scan — scripts/refresh.py checks ~1/7 of the catalog (one GET per game),
|
|
# read-only, and uploads a patch (checkpointed, deadline-bound).
|
|
# apply — applies the patch to the latest main and pushes (bash/commit_push.sh
|
|
# re-applies on a fresh main if the push races another writer).
|
|
# notify — Discord on failure / cancel / itch.io rate limit / mass change
|
|
# (a run that would remove or strike an implausible share of games
|
|
# — usually itch.io markup changes — withholds the removals).
|
|
#
|
|
# Also dispatched with `full: true` (by force_update.yml, or by hand): re-scrape
|
|
# every scraper-owned field of one URL or of the next batch.
|
|
name: Refresh catalog
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '47 2 * * *' # daily; GitHub may start scheduled runs hours late
|
|
workflow_dispatch:
|
|
inputs:
|
|
budget:
|
|
description: 'Games to check (0 = 1/7 of the catalog)'
|
|
type: number
|
|
default: 0
|
|
url:
|
|
description: 'Only this game URL (optional)'
|
|
type: string
|
|
required: false
|
|
allow_mass_removal:
|
|
description: 'Apply removals even if the mass-change guard trips (confirm on itch.io first)'
|
|
type: boolean
|
|
default: false
|
|
full:
|
|
description: 'Re-scrape every scraper-owned field (rotates on the last full re-scrape)'
|
|
type: boolean
|
|
default: false
|
|
|
|
permissions: {}
|
|
|
|
# One refresh pipeline (scan → apply) at a time, FIFO — a scheduled refresh, a
|
|
# manual one and a force-update batch never start from the same state (the scan
|
|
# checks out the branch as it is when the run starts), so an older removal can't
|
|
# undo a newer healthy check, and queued full batches move on through the catalog.
|
|
concurrency:
|
|
group: refresh-pipeline
|
|
cancel-in-progress: false
|
|
queue: max
|
|
|
|
env:
|
|
PYTHONUNBUFFERED: '1'
|
|
|
|
jobs:
|
|
scan:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 60
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
rate_limited: ${{ steps.refresh.outputs.rate_limited }}
|
|
mass_change: ${{ steps.refresh.outputs.mass_change }}
|
|
steps:
|
|
# The branch's latest commit, not the one it had when the run was queued:
|
|
# a run waiting behind another must see that run's refresh state.
|
|
- name: Checkout repository
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ github.ref }}
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
with:
|
|
python-version-file: .github/python-version
|
|
cache: pip
|
|
cache-dependency-path: requirements.txt
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
pip install --require-hashes --only-binary :all: -r requirements.txt
|
|
|
|
- name: Check a slice of the catalog
|
|
id: refresh
|
|
timeout-minutes: 55
|
|
env:
|
|
BUDGET: ${{ inputs.budget || 0 }}
|
|
URL: ${{ inputs.url }}
|
|
FULL: ${{ inputs.full && 'true' || 'false' }}
|
|
ALLOW_MASS_REMOVAL: ${{ inputs.allow_mass_removal && 'true' || 'false' }}
|
|
run: |
|
|
args=(--out "$RUNNER_TEMP/patch.json" --budget "$BUDGET" --deadline-min 45)
|
|
if [ "$FULL" = "true" ]; then args+=(--full); fi
|
|
if [ "$ALLOW_MASS_REMOVAL" = "true" ]; then args+=(--allow-mass-removal); fi
|
|
if [ -n "$URL" ]; then args+=(--url "$URL"); fi
|
|
python scripts/refresh.py "${args[@]}"
|
|
|
|
- name: Upload patch
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
with:
|
|
name: refresh-patch
|
|
path: ${{ runner.temp }}/patch.json
|
|
retention-days: 7
|
|
if-no-files-found: warn
|
|
|
|
apply:
|
|
needs: scan
|
|
# Also apply the checkpointed patch of a scan that failed or timed out
|
|
# (a timed-out step fails, it does not cancel) — but not of a cancelled run.
|
|
if: ${{ !cancelled() && needs.scan.result != 'skipped' }}
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: write
|
|
# FIFO: with the default queue a newer pending apply would replace (and
|
|
# discard) an older pending one.
|
|
concurrency:
|
|
group: refresh-apply
|
|
cancel-in-progress: false
|
|
queue: max
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # zizmor: ignore[artipacked] — commit_push.sh pushes with this token
|
|
with:
|
|
ref: main
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
with:
|
|
python-version-file: .github/python-version
|
|
cache: pip
|
|
cache-dependency-path: requirements.txt
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
pip install --require-hashes --only-binary :all: -r requirements.txt
|
|
|
|
- name: Download patch
|
|
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
|
with:
|
|
name: refresh-patch
|
|
path: ${{ runner.temp }}
|
|
|
|
- name: Apply, validate and push
|
|
env:
|
|
KIND: ${{ inputs.full && 'Force update' || 'Refresh catalog' }}
|
|
ALLOW_MASS_REMOVAL: ${{ inputs.allow_mass_removal && '1' || '0' }}
|
|
run: bash bash/commit_push.sh "$RUNNER_TEMP/patch.json" "$KIND [$(date -u +%F)]"
|
|
|
|
notify:
|
|
needs: [scan, apply]
|
|
if: >-
|
|
always() && (contains(needs.*.result, 'failure') ||
|
|
contains(needs.*.result, 'cancelled') || needs.scan.outputs.rate_limited == 'true' ||
|
|
needs.scan.outputs.mass_change == 'true')
|
|
permissions: {}
|
|
uses: ./.github/workflows/notify-failure.yml
|
|
with:
|
|
summary: >-
|
|
scan=${{ needs.scan.result }}, apply=${{ needs.apply.result }},
|
|
rate_limited=${{ needs.scan.outputs.rate_limited || 'false' }},
|
|
mass_change=${{ needs.scan.outputs.mass_change || 'false' }}
|
|
secrets:
|
|
DISCORD_CI_WEBHOOK: ${{ secrets.DISCORD_CI_WEBHOOK }}
|