chore(pipeline): retire Check Dead Links
It sent a HEAD request to every store page every five days, 5.5 hours at 9,847 games, and flagged 404/410. Steam sends a delisted app's page to the store front, which answers 200: scripts/dead_links.log never existed and no game ever became "delisted". The purge's GetItems screen sees delisted games (unlisted) instead. Removes the workflow, bash/dead_links.sh, scripts/check_dead_links.py and DEAD_LINKS_LOG, and the workflow from web/shared/pipeline.ts and the failure notifier's list. check_store_page() stays: it is the health check's last resort. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1da4d5940e
commit
6e7d1bcf15
10 files changed
+11
-85
No files matched your search
@@ -1,19 +0,0 @@
|
||||
name : Check Dead Links
|
||||
on :
|
||||
schedule : [ { cron : '0 4 1,6,11,16,21,26 * *' } ]
|
||||
workflow_dispatch :
|
||||
permissions : { contents : write }
|
||||
concurrency :
|
||||
group : data-write
|
||||
cancel-in-progress : false
|
||||
jobs :
|
||||
check :
|
||||
runs-on : ubuntu-latest
|
||||
steps :
|
||||
-
|
||||
uses : actions/checkout@v6
|
||||
-
|
||||
uses : actions/setup-python@v6
|
||||
with : { python-version : '3.15' }
|
||||
-
|
||||
run : chmod +x ./bash/dead_links.sh && ./bash/dead_links.sh
|
||||
@@ -13,7 +13,6 @@ on:
|
||||
- "Auto Update JSON Data"
|
||||
- "Top Online Leaderboard"
|
||||
- "Update Reviews"
|
||||
- "Check Dead Links"
|
||||
- "Purge Unhealthy Games"
|
||||
- "Ingest New Game Links"
|
||||
- "Force Re-fetch All (v2.1 Schema)"
|
||||
|
||||
@@ -13,8 +13,6 @@ Thumbs.db
|
||||
# scripts/mark_dead_games.py and explicitly staged by mark-dead-games.yml.
|
||||
# It survives *.log today only because ignore rules do not apply to
|
||||
# already-tracked files -- rename or recreate it and it silently vanishes.
|
||||
# (scripts/dead_links.log is deliberately NOT excepted: bash/dead_links.sh
|
||||
# runs `git add .`, and that log is local diagnostic output.)
|
||||
!scripts/dead_games.log
|
||||
*.tmp
|
||||
*.temp
|
||||
|
||||
+6
-7
@@ -1,13 +1,12 @@
|
||||
#!/bin/bash
|
||||
# Commit what a data job changed, and push it even if main moved meanwhile.
|
||||
#
|
||||
# A data job can run for hours (update-reviews, check-dead-links,
|
||||
# purge-unhealthy), and `concurrency: data-write` serialises only the
|
||||
# WORKFLOWS. main still moves underneath one: a merged pull request, an /admin
|
||||
# approval, removal or override the Worker commits. A plain `git push` is then
|
||||
# rejected ("fetch first")
|
||||
# and the whole run is thrown away - purge-unhealthy lost four hours of work
|
||||
# that way on 2026-09-21.
|
||||
# A data job can run for an hour or more (refresh-store-data, the
|
||||
# leaderboards), and `concurrency: data-write` serialises only the WORKFLOWS.
|
||||
# main still moves underneath one: a merged pull request, an /admin approval,
|
||||
# removal or override the Worker commits. A plain `git push` is then rejected
|
||||
# ("fetch first") and the whole run is thrown away - purge-unhealthy lost four
|
||||
# hours of work that way on 2026-09-21.
|
||||
#
|
||||
# None of those other writers touch the shards, so rebasing onto them and
|
||||
# pushing again is safe. A genuine conflict still fails the job, loudly.
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
pip install --quiet --require-hashes --only-binary :all: -r requirements.txt
|
||||
python scripts/check_dead_links.py
|
||||
bash "$(dirname "$0")/commit_push.sh" "Dead link check [$(date +'%Y-%m-%d')]"
|
||||
@@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dead link checker – HEAD requests only."""
|
||||
import sys, os, random, time
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from datetime import datetime, timezone
|
||||
from core.constants import DEAD_LINKS_LOG, BATCH_SIZE, BATCH_PAUSE_MIN, BATCH_PAUSE_MAX
|
||||
from core.data_store import load_main, save_main, extract_appid, now_iso, append_note_idempotent
|
||||
from core.steam_client import get_client
|
||||
|
||||
def main():
|
||||
games = load_main()
|
||||
active = [g for g in games if g.get("status", "active") == "active"]
|
||||
print(f"Checking {len(active)} games...\n")
|
||||
client, dead, total = get_client(), [], len(active)
|
||||
for bs in range(0, total, BATCH_SIZE):
|
||||
be = min(bs + BATCH_SIZE, total)
|
||||
print(f"── Batch {bs//BATCH_SIZE+1} ──")
|
||||
for i in range(bs, be):
|
||||
g = active[i]
|
||||
appid = extract_appid(g.get("link", ""))
|
||||
if not appid: continue
|
||||
name = g.get("name", appid)
|
||||
code = client.check_store_page(appid)
|
||||
if code in (404, 410):
|
||||
print(f" [{i+1}/{total}] 💀 {name} ({code})")
|
||||
g["status"] = "delisted"
|
||||
append_note_idempotent(g, "💀 Delisted", f"💀 Delisted ({code})")
|
||||
g["last_updated"] = now_iso()
|
||||
dead.append({"name": name, "appid": appid, "code": code})
|
||||
else:
|
||||
print(f" [{i+1}/{total}] {'✓' if code==200 else f'⚠{code}'} {name[:50]}")
|
||||
if be < total:
|
||||
time.sleep(random.uniform(BATCH_PAUSE_MIN, BATCH_PAUSE_MAX))
|
||||
save_main(games)
|
||||
if dead:
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
with open(DEAD_LINKS_LOG, "a", encoding="utf-8") as f:
|
||||
f.write(f"\n── {ts} ──\n")
|
||||
for d in dead: f.write(f" [{d['code']}] {d['name']} ({d['appid']})\n")
|
||||
print(f"\n💀 {len(dead)} dead")
|
||||
else:
|
||||
print(f"\n✓ All {total} alive!")
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
@@ -7,7 +7,6 @@ import os
|
||||
DATA_DIR = "data"
|
||||
LEGACY_JSONL = "scripts/data.jsonl" # fallback pre-migration
|
||||
TEMP_JSONL = "scripts/temp_info.jsonl"
|
||||
DEAD_LINKS_LOG = "scripts/dead_links.log"
|
||||
REMOVED_JSONL = "scripts/removed_games.jsonl"
|
||||
GAMES_DIR = "games"
|
||||
OVERRIDES_DIR = "data/overrides" # one <appid>.json per human-edited game
|
||||
|
||||
@@ -442,7 +442,7 @@ _SKELETON_TEMPLATE = {
|
||||
def append_note_idempotent(game: dict, marker: str, full_note: str) -> bool:
|
||||
"""Append `full_note` to game['notes'] only if `marker` isn't already present.
|
||||
|
||||
Used by check_dead_links and mark_dead_games to avoid duplicating system
|
||||
Used by mark_dead_games to avoid duplicating system
|
||||
notes when a workflow re-runs over an already-marked game. Returns True
|
||||
when the note was actually changed.
|
||||
"""
|
||||
|
||||
@@ -97,7 +97,7 @@ def preserve_machine_notes(current: str, human: str) -> str:
|
||||
"""Re-attach machine-appended markers that the human value would drop.
|
||||
|
||||
`notes` is a hybrid field: the pipeline appends its own segments to it —
|
||||
check_dead_links adds "Delisted", mark_dead_games adds "Dead game",
|
||||
the retired dead-link check added "Delisted", mark_dead_games adds "Dead game",
|
||||
fetcher adds "No longer free!". Those appends are one-way. mark_dead_games
|
||||
only calls append_note_idempotent() when `is_dead` is still False, so once
|
||||
a game is marked, a stripped marker is NEVER re-appended. A plain
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
* the GitHub API), so renaming a workflow does not break the page.
|
||||
*
|
||||
* `dataWrite` marks the workflows in `concurrency: {group: data-write}`.
|
||||
* Auto-approve commits only while none of them is running or waiting: the
|
||||
* group keeps ONE pending run, and the ingest an approval starts would evict a
|
||||
* job already waiting (pipeline.test.ts holds this list to the YAML).
|
||||
* Auto-approve commits only while none of them is running or waiting, so the
|
||||
* ingest an approval starts can begin at once (pipeline.test.ts holds this
|
||||
* list to the YAML, and every one of them to `queue: max`).
|
||||
*
|
||||
* Pure: no DOM, no Env, no I/O.
|
||||
*/
|
||||
@@ -24,7 +24,6 @@ export const DATA_WORKFLOWS: DataWorkflow[] = [
|
||||
{ file: ".github/workflows/update-reviews.yml", label: "Update reviews", dataWrite: true },
|
||||
{ file: ".github/workflows/top-online.yml", label: "Top online", dataWrite: true },
|
||||
{ file: ".github/workflows/top-offline.yml", label: "Top offline", dataWrite: true },
|
||||
{ file: ".github/workflows/check-dead-links.yml", label: "Check dead links", dataWrite: true },
|
||||
{ file: ".github/workflows/mark-dead-games.yml", label: "Mark dead games", dataWrite: true },
|
||||
{ file: ".github/workflows/purge-unhealthy.yml", label: "Purge unhealthy games", dataWrite: true },
|
||||
{ file: ".github/workflows/snapshot-daily.yml", label: "Daily snapshot", dataWrite: true },
|
||||
|
||||
Reference in new issue
Block a user