From 6e7d1bcf153a308bef6adbebfb21b778c9180077 Mon Sep 17 00:00:00 2001 From: poli0981 <127664709+poli0981@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:22:08 +0700 Subject: [PATCH] chore(pipeline): retire Check Dead Links It sent a HEAD request to every store page every five days, 5.5 hours at 9,847 games, and flagged 404/410. Steam sends a delisted app's page to the store front, which answers 200: scripts/dead_links.log never existed and no game ever became "delisted". The purge's GetItems screen sees delisted games (unlisted) instead. Removes the workflow, bash/dead_links.sh, scripts/check_dead_links.py and DEAD_LINKS_LOG, and the workflow from web/shared/pipeline.ts and the failure notifier's list. check_store_page() stays: it is the health check's last resort. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/check-dead-links.yml | 19 ----------- .github/workflows/notify-ci-failure.yml | 1 - .gitignore | 2 -- bash/commit_push.sh | 13 ++++---- bash/dead_links.sh | 5 --- scripts/check_dead_links.py | 44 ------------------------- scripts/core/constants.py | 1 - scripts/core/data_store.py | 2 +- scripts/core/overrides.py | 2 +- web/shared/pipeline.ts | 7 ++-- 10 files changed, 11 insertions(+), 85 deletions(-) delete mode 100644 .github/workflows/check-dead-links.yml delete mode 100755 bash/dead_links.sh delete mode 100644 scripts/check_dead_links.py diff --git a/.github/workflows/check-dead-links.yml b/.github/workflows/check-dead-links.yml deleted file mode 100644 index fcfdad73..00000000 --- a/.github/workflows/check-dead-links.yml +++ /dev/null @@ -1,19 +0,0 @@ -name : Check Dead Links -on : - schedule : [ { cron : '0 4 1,6,11,16,21,26 * *' } ] - workflow_dispatch : -permissions : { contents : write } -concurrency : - group : data-write - cancel-in-progress : false -jobs : - check : - runs-on : ubuntu-latest - steps : - - - uses : actions/checkout@v6 - - - uses : actions/setup-python@v6 - with : { python-version : '3.15' } - - - run : chmod +x ./bash/dead_links.sh && ./bash/dead_links.sh diff --git a/.github/workflows/notify-ci-failure.yml b/.github/workflows/notify-ci-failure.yml index 8bc72ba7..360974da 100644 --- a/.github/workflows/notify-ci-failure.yml +++ b/.github/workflows/notify-ci-failure.yml @@ -13,7 +13,6 @@ on: - "Auto Update JSON Data" - "Top Online Leaderboard" - "Update Reviews" - - "Check Dead Links" - "Purge Unhealthy Games" - "Ingest New Game Links" - "Force Re-fetch All (v2.1 Schema)" diff --git a/.gitignore b/.gitignore index 517dfc9c..f060077d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,6 @@ Thumbs.db # scripts/mark_dead_games.py and explicitly staged by mark-dead-games.yml. # It survives *.log today only because ignore rules do not apply to # already-tracked files -- rename or recreate it and it silently vanishes. -# (scripts/dead_links.log is deliberately NOT excepted: bash/dead_links.sh -# runs `git add .`, and that log is local diagnostic output.) !scripts/dead_games.log *.tmp *.temp diff --git a/bash/commit_push.sh b/bash/commit_push.sh index f81924f3..4a77db71 100755 --- a/bash/commit_push.sh +++ b/bash/commit_push.sh @@ -1,13 +1,12 @@ #!/bin/bash # Commit what a data job changed, and push it even if main moved meanwhile. # -# A data job can run for hours (update-reviews, check-dead-links, -# purge-unhealthy), and `concurrency: data-write` serialises only the -# WORKFLOWS. main still moves underneath one: a merged pull request, an /admin -# approval, removal or override the Worker commits. A plain `git push` is then -# rejected ("fetch first") -# and the whole run is thrown away - purge-unhealthy lost four hours of work -# that way on 2026-09-21. +# A data job can run for an hour or more (refresh-store-data, the +# leaderboards), and `concurrency: data-write` serialises only the WORKFLOWS. +# main still moves underneath one: a merged pull request, an /admin approval, +# removal or override the Worker commits. A plain `git push` is then rejected +# ("fetch first") and the whole run is thrown away - purge-unhealthy lost four +# hours of work that way on 2026-09-21. # # None of those other writers touch the shards, so rebasing onto them and # pushing again is safe. A genuine conflict still fails the job, loudly. diff --git a/bash/dead_links.sh b/bash/dead_links.sh deleted file mode 100755 index a62b4486..00000000 --- a/bash/dead_links.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -euo pipefail -pip install --quiet --require-hashes --only-binary :all: -r requirements.txt -python scripts/check_dead_links.py -bash "$(dirname "$0")/commit_push.sh" "Dead link check [$(date +'%Y-%m-%d')]" diff --git a/scripts/check_dead_links.py b/scripts/check_dead_links.py deleted file mode 100644 index b816ad09..00000000 --- a/scripts/check_dead_links.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 -"""Dead link checker – HEAD requests only.""" -import sys, os, random, time -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from datetime import datetime, timezone -from core.constants import DEAD_LINKS_LOG, BATCH_SIZE, BATCH_PAUSE_MIN, BATCH_PAUSE_MAX -from core.data_store import load_main, save_main, extract_appid, now_iso, append_note_idempotent -from core.steam_client import get_client - -def main(): - games = load_main() - active = [g for g in games if g.get("status", "active") == "active"] - print(f"Checking {len(active)} games...\n") - client, dead, total = get_client(), [], len(active) - for bs in range(0, total, BATCH_SIZE): - be = min(bs + BATCH_SIZE, total) - print(f"── Batch {bs//BATCH_SIZE+1} ──") - for i in range(bs, be): - g = active[i] - appid = extract_appid(g.get("link", "")) - if not appid: continue - name = g.get("name", appid) - code = client.check_store_page(appid) - if code in (404, 410): - print(f" [{i+1}/{total}] 💀 {name} ({code})") - g["status"] = "delisted" - append_note_idempotent(g, "💀 Delisted", f"💀 Delisted ({code})") - g["last_updated"] = now_iso() - dead.append({"name": name, "appid": appid, "code": code}) - else: - print(f" [{i+1}/{total}] {'✓' if code==200 else f'⚠{code}'} {name[:50]}") - if be < total: - time.sleep(random.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX)) - save_main(games) - if dead: - ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") - with open(DEAD_LINKS_LOG, "a", encoding="utf-8") as f: - f.write(f"\n── {ts} ──\n") - for d in dead: f.write(f" [{d['code']}] {d['name']} ({d['appid']})\n") - print(f"\n💀 {len(dead)} dead") - else: - print(f"\n✓ All {total} alive!") - -if __name__ == "__main__": main() diff --git a/scripts/core/constants.py b/scripts/core/constants.py index 00933a86..50e32d2b 100644 --- a/scripts/core/constants.py +++ b/scripts/core/constants.py @@ -7,7 +7,6 @@ import os DATA_DIR = "data" LEGACY_JSONL = "scripts/data.jsonl" # fallback pre-migration TEMP_JSONL = "scripts/temp_info.jsonl" -DEAD_LINKS_LOG = "scripts/dead_links.log" REMOVED_JSONL = "scripts/removed_games.jsonl" GAMES_DIR = "games" OVERRIDES_DIR = "data/overrides" # one .json per human-edited game diff --git a/scripts/core/data_store.py b/scripts/core/data_store.py index 89910f7a..07260ef0 100644 --- a/scripts/core/data_store.py +++ b/scripts/core/data_store.py @@ -442,7 +442,7 @@ _SKELETON_TEMPLATE = { def append_note_idempotent(game: dict, marker: str, full_note: str) -> bool: """Append `full_note` to game['notes'] only if `marker` isn't already present. - Used by check_dead_links and mark_dead_games to avoid duplicating system + Used by mark_dead_games to avoid duplicating system notes when a workflow re-runs over an already-marked game. Returns True when the note was actually changed. """ diff --git a/scripts/core/overrides.py b/scripts/core/overrides.py index 92518af8..dbfbcffc 100644 --- a/scripts/core/overrides.py +++ b/scripts/core/overrides.py @@ -97,7 +97,7 @@ def preserve_machine_notes(current: str, human: str) -> str: """Re-attach machine-appended markers that the human value would drop. `notes` is a hybrid field: the pipeline appends its own segments to it — - check_dead_links adds "Delisted", mark_dead_games adds "Dead game", + the retired dead-link check added "Delisted", mark_dead_games adds "Dead game", fetcher adds "No longer free!". Those appends are one-way. mark_dead_games only calls append_note_idempotent() when `is_dead` is still False, so once a game is marked, a stripped marker is NEVER re-appended. A plain diff --git a/web/shared/pipeline.ts b/web/shared/pipeline.ts index dade09b5..8048b54d 100644 --- a/web/shared/pipeline.ts +++ b/web/shared/pipeline.ts @@ -3,9 +3,9 @@ * the GitHub API), so renaming a workflow does not break the page. * * `dataWrite` marks the workflows in `concurrency: {group: data-write}`. - * Auto-approve commits only while none of them is running or waiting: the - * group keeps ONE pending run, and the ingest an approval starts would evict a - * job already waiting (pipeline.test.ts holds this list to the YAML). + * Auto-approve commits only while none of them is running or waiting, so the + * ingest an approval starts can begin at once (pipeline.test.ts holds this + * list to the YAML, and every one of them to `queue: max`). * * Pure: no DOM, no Env, no I/O. */ @@ -24,7 +24,6 @@ export const DATA_WORKFLOWS: DataWorkflow[] = [ { file: ".github/workflows/update-reviews.yml", label: "Update reviews", dataWrite: true }, { file: ".github/workflows/top-online.yml", label: "Top online", dataWrite: true }, { file: ".github/workflows/top-offline.yml", label: "Top offline", dataWrite: true }, - { file: ".github/workflows/check-dead-links.yml", label: "Check dead links", dataWrite: true }, { file: ".github/workflows/mark-dead-games.yml", label: "Mark dead games", dataWrite: true }, { file: ".github/workflows/purge-unhealthy.yml", label: "Purge unhealthy games", dataWrite: true }, { file: ".github/workflows/snapshot-daily.yml", label: "Daily snapshot", dataWrite: true },