chore(pipeline): retire Check Dead Links

It sent a HEAD request to every store page every five days, 5.5 hours at
9,847 games, and flagged 404/410. Steam sends a delisted app's page to
the store front, which answers 200: scripts/dead_links.log never existed
and no game ever became "delisted". The purge's GetItems screen sees
delisted games (unlisted) instead.

Removes the workflow, bash/dead_links.sh, scripts/check_dead_links.py
and DEAD_LINKS_LOG, and the workflow from web/shared/pipeline.ts and the
failure notifier's list. check_store_page() stays: it is the health
check's last resort.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
poli0981andClaude Opus 5.5 committed 2026-10-23 22:22:08 +07:00
1 parent 1da4d5940e
commit 6e7d1bcf15
10 files changed
+11 -85

No files matched your search

-19
View File
@@ -1,19 +0,0 @@
name : Check Dead Links
on :
schedule : [ { cron : '0 4 1,6,11,16,21,26 * *' } ]
workflow_dispatch :
permissions : { contents : write }
concurrency :
group : data-write
cancel-in-progress : false
jobs :
check :
runs-on : ubuntu-latest
steps :
-
uses : actions/checkout@v6
-
uses : actions/setup-python@v6
with : { python-version : '3.15' }
-
run : chmod +x ./bash/dead_links.sh && ./bash/dead_links.sh
-1
View File
@@ -13,7 +13,6 @@ on:
- "Auto Update JSON Data" - "Auto Update JSON Data"
- "Top Online Leaderboard" - "Top Online Leaderboard"
- "Update Reviews" - "Update Reviews"
- "Check Dead Links"
- "Purge Unhealthy Games" - "Purge Unhealthy Games"
- "Ingest New Game Links" - "Ingest New Game Links"
- "Force Re-fetch All (v2.1 Schema)" - "Force Re-fetch All (v2.1 Schema)"
-2
View File
@@ -13,8 +13,6 @@ Thumbs.db
# scripts/mark_dead_games.py and explicitly staged by mark-dead-games.yml. # scripts/mark_dead_games.py and explicitly staged by mark-dead-games.yml.
# It survives *.log today only because ignore rules do not apply to # It survives *.log today only because ignore rules do not apply to
# already-tracked files -- rename or recreate it and it silently vanishes. # already-tracked files -- rename or recreate it and it silently vanishes.
# (scripts/dead_links.log is deliberately NOT excepted: bash/dead_links.sh
# runs `git add .`, and that log is local diagnostic output.)
!scripts/dead_games.log !scripts/dead_games.log
*.tmp *.tmp
*.temp *.temp
+6 -7
View File
@@ -1,13 +1,12 @@
#!/bin/bash #!/bin/bash
# Commit what a data job changed, and push it even if main moved meanwhile. # Commit what a data job changed, and push it even if main moved meanwhile.
# #
# A data job can run for hours (update-reviews, check-dead-links, # A data job can run for an hour or more (refresh-store-data, the
# purge-unhealthy), and `concurrency: data-write` serialises only the # leaderboards), and `concurrency: data-write` serialises only the WORKFLOWS.
# WORKFLOWS. main still moves underneath one: a merged pull request, an /admin # main still moves underneath one: a merged pull request, an /admin approval,
# approval, removal or override the Worker commits. A plain `git push` is then # removal or override the Worker commits. A plain `git push` is then rejected
# rejected ("fetch first") # ("fetch first") and the whole run is thrown away - purge-unhealthy lost four
# and the whole run is thrown away - purge-unhealthy lost four hours of work # hours of work that way on 2026-09-21.
# that way on 2026-09-21.
# #
# None of those other writers touch the shards, so rebasing onto them and # None of those other writers touch the shards, so rebasing onto them and
# pushing again is safe. A genuine conflict still fails the job, loudly. # pushing again is safe. A genuine conflict still fails the job, loudly.
-5
View File
@@ -1,5 +0,0 @@
#!/bin/bash
set -euo pipefail
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
python scripts/check_dead_links.py
bash "$(dirname "$0")/commit_push.sh" "Dead link check [$(date +'%Y-%m-%d')]"
-44
View File
@@ -1,44 +0,0 @@
#!/usr/bin/env python3
"""Dead link checker – HEAD requests only."""
import sys, os, random, time
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from datetime import datetime, timezone
from core.constants import DEAD_LINKS_LOG, BATCH_SIZE, BATCH_PAUSE_MIN, BATCH_PAUSE_MAX
from core.data_store import load_main, save_main, extract_appid, now_iso, append_note_idempotent
from core.steam_client import get_client
def main():
games = load_main()
active = [g for g in games if g.get("status", "active") == "active"]
print(f"Checking {len(active)} games...\n")
client, dead, total = get_client(), [], len(active)
for bs in range(0, total, BATCH_SIZE):
be = min(bs + BATCH_SIZE, total)
print(f"── Batch {bs//BATCH_SIZE+1} ──")
for i in range(bs, be):
g = active[i]
appid = extract_appid(g.get("link", ""))
if not appid: continue
name = g.get("name", appid)
code = client.check_store_page(appid)
if code in (404, 410):
print(f" [{i+1}/{total}] 💀 {name} ({code})")
g["status"] = "delisted"
append_note_idempotent(g, "💀 Delisted", f"💀 Delisted ({code})")
g["last_updated"] = now_iso()
dead.append({"name": name, "appid": appid, "code": code})
else:
print(f" [{i+1}/{total}] {'✓' if code==200 else f'⚠{code}'} {name[:50]}")
if be < total:
time.sleep(random.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX))
save_main(games)
if dead:
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
with open(DEAD_LINKS_LOG, "a", encoding="utf-8") as f:
f.write(f"\n── {ts} ──\n")
for d in dead: f.write(f" [{d['code']}] {d['name']} ({d['appid']})\n")
print(f"\n💀 {len(dead)} dead")
else:
print(f"\n✓ All {total} alive!")
if __name__ == "__main__": main()
-1
View File
@@ -7,7 +7,6 @@ import os
DATA_DIR = "data" DATA_DIR = "data"
LEGACY_JSONL = "scripts/data.jsonl" # fallback pre-migration LEGACY_JSONL = "scripts/data.jsonl" # fallback pre-migration
TEMP_JSONL = "scripts/temp_info.jsonl" TEMP_JSONL = "scripts/temp_info.jsonl"
DEAD_LINKS_LOG = "scripts/dead_links.log"
REMOVED_JSONL = "scripts/removed_games.jsonl" REMOVED_JSONL = "scripts/removed_games.jsonl"
GAMES_DIR = "games" GAMES_DIR = "games"
OVERRIDES_DIR = "data/overrides" # one <appid>.json per human-edited game OVERRIDES_DIR = "data/overrides" # one <appid>.json per human-edited game
+1 -1
View File
@@ -442,7 +442,7 @@ _SKELETON_TEMPLATE = {
def append_note_idempotent(game: dict, marker: str, full_note: str) -> bool: def append_note_idempotent(game: dict, marker: str, full_note: str) -> bool:
"""Append `full_note` to game['notes'] only if `marker` isn't already present. """Append `full_note` to game['notes'] only if `marker` isn't already present.
Used by check_dead_links and mark_dead_games to avoid duplicating system Used by mark_dead_games to avoid duplicating system
notes when a workflow re-runs over an already-marked game. Returns True notes when a workflow re-runs over an already-marked game. Returns True
when the note was actually changed. when the note was actually changed.
""" """
+1 -1
View File
@@ -97,7 +97,7 @@ def preserve_machine_notes(current: str, human: str) -> str:
"""Re-attach machine-appended markers that the human value would drop. """Re-attach machine-appended markers that the human value would drop.
`notes` is a hybrid field: the pipeline appends its own segments to it — `notes` is a hybrid field: the pipeline appends its own segments to it —
check_dead_links adds "Delisted", mark_dead_games adds "Dead game", the retired dead-link check added "Delisted", mark_dead_games adds "Dead game",
fetcher adds "No longer free!". Those appends are one-way. mark_dead_games fetcher adds "No longer free!". Those appends are one-way. mark_dead_games
only calls append_note_idempotent() when `is_dead` is still False, so once only calls append_note_idempotent() when `is_dead` is still False, so once
a game is marked, a stripped marker is NEVER re-appended. A plain a game is marked, a stripped marker is NEVER re-appended. A plain
+3 -4
View File
@@ -3,9 +3,9 @@
* the GitHub API), so renaming a workflow does not break the page. * the GitHub API), so renaming a workflow does not break the page.
* *
* `dataWrite` marks the workflows in `concurrency: {group: data-write}`. * `dataWrite` marks the workflows in `concurrency: {group: data-write}`.
* Auto-approve commits only while none of them is running or waiting: the * Auto-approve commits only while none of them is running or waiting, so the
* group keeps ONE pending run, and the ingest an approval starts would evict a * ingest an approval starts can begin at once (pipeline.test.ts holds this
* job already waiting (pipeline.test.ts holds this list to the YAML). * list to the YAML, and every one of them to `queue: max`).
* *
* Pure: no DOM, no Env, no I/O. * Pure: no DOM, no Env, no I/O.
*/ */
@@ -24,7 +24,6 @@ export const DATA_WORKFLOWS: DataWorkflow[] = [
{ file: ".github/workflows/update-reviews.yml", label: "Update reviews", dataWrite: true }, { file: ".github/workflows/update-reviews.yml", label: "Update reviews", dataWrite: true },
{ file: ".github/workflows/top-online.yml", label: "Top online", dataWrite: true }, { file: ".github/workflows/top-online.yml", label: "Top online", dataWrite: true },
{ file: ".github/workflows/top-offline.yml", label: "Top offline", dataWrite: true }, { file: ".github/workflows/top-offline.yml", label: "Top offline", dataWrite: true },
{ file: ".github/workflows/check-dead-links.yml", label: "Check dead links", dataWrite: true },
{ file: ".github/workflows/mark-dead-games.yml", label: "Mark dead games", dataWrite: true }, { file: ".github/workflows/mark-dead-games.yml", label: "Mark dead games", dataWrite: true },
{ file: ".github/workflows/purge-unhealthy.yml", label: "Purge unhealthy games", dataWrite: true }, { file: ".github/workflows/purge-unhealthy.yml", label: "Purge unhealthy games", dataWrite: true },
{ file: ".github/workflows/snapshot-daily.yml", label: "Daily snapshot", dataWrite: true }, { file: ".github/workflows/snapshot-daily.yml", label: "Daily snapshot", dataWrite: true },