release(v3.4.0): periodic refresh workflows + UTF-8 round-trip fix (#42)
- New scripts/update_reviews.py + bash + workflow on cron '0 5 1,15 * *':
refresh rating + rating_count fortnightly via scrape_game_info, skip on
network/parse errors so old values aren't blanked.
- New scripts/update_status.py + bash + workflow on cron '0 6 1 * *':
refresh status monthly using the same pattern.
- New scripts/force_update.py + bash + workflow (workflow_dispatch only):
optional url input; empty input re-scrapes every game. Always preserves
safe_virus / notes / nsfw per CLAUDE.md "Editable vs read-only fields".
Doubles as the canonical mojibake repair tool.
- Chain generate_table.yml on the three new workflows so lists/*.md regen
automatically after data mutations.
- Fix UTF-8 mojibake on webapp writes: webapp/src/lib/github/contents.ts
readFileWithSha and git-data.ts bulkDeleteGames decoded GitHub's base64
content with atob alone (Latin-1 binary string). Multi-byte UTF-8 sequences
became mojibake; next write re-encoded them as UTF-8, double-corrupting per
edit cycle. Confirmed damage in data_game/game_info_002.json
('BotÃÂão Esquerdo' was 'Botão Esquerdo'). Extract shared helper
webapp/src/lib/github/encoding.ts (base64ToUtf8 / utf8ToBase64) using
TextDecoder('utf-8') / TextEncoder; route both call sites through it.
- Version bumps: README + README.vi -> 3.4.0, about.ts APP.version -> 3.4.0,
new CHANGELOG entry, CLAUDE.md GitHub Actions table + gotcha #8 updated.
- No Rust binary change: tauri Cargo.toml / tauri.conf.json untouched.
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c90e038585
commit
500ff7fd02
18 files changed
+461
-14
No files matched your search
@@ -0,0 +1,34 @@
|
||||
name: Force update
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
url:
|
||||
description: 'itch.io URL to re-scrape (optional; empty = re-scrape ALL games)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
force-update:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
token: ${{ secrets.GH_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.14'
|
||||
|
||||
- name: Force re-scrape (preserve safe_virus / notes / nsfw)
|
||||
env:
|
||||
INPUT_URL: ${{ inputs.url }}
|
||||
run: |
|
||||
chmod +x ./bash/force_update.sh
|
||||
./bash/force_update.sh
|
||||
shell: bash
|
||||
@@ -5,6 +5,9 @@ on:
|
||||
- 'Update JSON file game information'
|
||||
- 'Check paid games'
|
||||
- 'Check dead game links'
|
||||
- 'Update reviews'
|
||||
- 'Update status'
|
||||
- 'Force update'
|
||||
types: [completed]
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
name: Update reviews
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 5 1,15 * *' # 05:00 UTC on the 1st and 15th of each month
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
update-reviews:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
token: ${{ secrets.GH_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.14'
|
||||
|
||||
- name: Refresh rating + rating_count
|
||||
run: |
|
||||
chmod +x ./bash/update_reviews.sh
|
||||
./bash/update_reviews.sh
|
||||
shell: bash
|
||||
@@ -0,0 +1,29 @@
|
||||
name: Update status
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 1 * *' # 06:00 UTC on the 1st of each month
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
update-status:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
token: ${{ secrets.GH_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.14'
|
||||
|
||||
- name: Refresh status field
|
||||
run: |
|
||||
chmod +x ./bash/update_status.sh
|
||||
./bash/update_status.sh
|
||||
shell: bash
|
||||
@@ -2,6 +2,27 @@
|
||||
|
||||
All notable changes to this project will be documented here.
|
||||
|
||||
## [3.4.0] - 2026-05-16 (Periodic refresh workflows + UTF-8 round-trip fix)
|
||||
|
||||
### Added
|
||||
|
||||
- **Update reviews — bi-weekly workflow.** New [`scripts/update_reviews.py`](scripts/update_reviews.py), wrapper [`bash/update_reviews.sh`](bash/update_reviews.sh), and [`.github/workflows/update_reviews.yml`](.github/workflows/update_reviews.yml) re-scrape every existing game's itch.io page on the 1st and 15th of each month (05:00 UTC) and write fresh `rating` + `rating_count`. Network errors / 404 keep the old values. Touches only those two fields — everything else (annotations, metadata) is left alone.
|
||||
- **Update status — monthly workflow.** New [`scripts/update_status.py`](scripts/update_status.py), [`bash/update_status.sh`](bash/update_status.sh), and [`.github/workflows/update_status.yml`](.github/workflows/update_status.yml) refresh the `status` field on the 1st of each month (06:00 UTC). Same skip-on-error semantics.
|
||||
- **Force update — manual emergency button.** New [`scripts/force_update.py`](scripts/force_update.py), [`bash/force_update.sh`](bash/force_update.sh), and [`.github/workflows/force_update.yml`](.github/workflows/force_update.yml) accept an optional `url` input. Empty input re-scrapes every game; a single URL targets just that one. `workflow_dispatch` only — no schedule. Every run preserves the three user-editable annotations (`safe_virus`, `notes`, `nsfw`) per the read/write field convention in [CLAUDE.md](CLAUDE.md). Doubles as the canonical repair tool for mojibake left by older webapp commits — re-scrape pulls clean UTF-8 from itch.io.
|
||||
- **`generate_table.yml` chained on the three new workflows** so `lists/*.md` stay in sync after any data mutation.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **UTF-8 mojibake on webapp commits.** [`webapp/src/lib/github/contents.ts`](webapp/src/lib/github/contents.ts) `readFileWithSha` and [`webapp/src/lib/github/git-data.ts`](webapp/src/lib/github/git-data.ts) `bulkDeleteGames` were decoding GitHub's base64 content with `atob` alone, producing a Latin-1 binary string. Multi-byte UTF-8 sequences (e.g. Vietnamese `ã` = `0xC3 0xA3`) became two Latin-1 code points (`ã`); the next write re-encoded them as UTF-8 bytes, double-corrupting on every edit. Confirmed live damage in `data_game/game_info_002.json` (`BotÃÂão Esquerdo` was originally `Botão Esquerdo`). Extracted a shared helper [`webapp/src/lib/github/encoding.ts`](webapp/src/lib/github/encoding.ts) (`base64ToUtf8` / `utf8ToBase64`) that uses `TextDecoder('utf-8')` / `TextEncoder` and routes both call sites through it. Write path was already correct.
|
||||
|
||||
### Notes
|
||||
|
||||
- **Repair existing mojibake**: after this release deploys, run `Force update` once via `workflow_dispatch` with no input. It re-scrapes every game (~15–25 min) and writes back clean UTF-8 while preserving manual annotations. `generate_table.yml` will then auto-regenerate the markdown lists.
|
||||
- **Browser cache**: anyone editing via the webapp should hard-refresh once after the new build lands so they're not using the pre-fix bundle (which would continue to mangle non-Latin chars on the next edit).
|
||||
- **README + README.vi badges** bumped to `3.4.0`. No Tauri / Rust binary change — `Cargo.toml` and `tauri.conf.json` stay at `0.1.1`.
|
||||
|
||||
---
|
||||
|
||||
## [3.3.0] - 2026-05-16 (Mobile polish + Settings + in-browser GPG commit signing)
|
||||
|
||||
### Added
|
||||
|
||||
@@ -54,8 +54,11 @@ Of the 23 fields per game, only **3** are user-editable: `safe_virus`, `notes`,
|
||||
|---|---|---|
|
||||
| `update.yml` | Daily 03:00 UTC + `workflow_dispatch` (with optional `url` input) | Reads `INPUT_URL` env var; falls back to `scripts/temp_link.json`. |
|
||||
| `check_paid.yml` | Every 2 days 04:00 UTC | |
|
||||
| `update_reviews.yml` | 1st + 15th of each month, 05:00 UTC + `workflow_dispatch` | Refresh `rating` + `rating_count`. Skip-on-error keeps old values. |
|
||||
| `update_status.yml` | 1st of each month, 06:00 UTC + `workflow_dispatch` | Refresh `status` only. Same skip semantics. |
|
||||
| `check_alive.yml` | Every 2 days 07:00 UTC | |
|
||||
| `generate_table.yml` | After update / check workflows succeed | Rebuilds `lists/*.md`. |
|
||||
| `force_update.yml` | `workflow_dispatch` ONLY (optional `url` input) | Emergency re-scrape. Empty input = re-scrape all. Always preserves `safe_virus` / `notes` / `nsfw`. Canonical tool for repairing mojibake. |
|
||||
| `generate_table.yml` | After update / check / refresh workflows succeed | Rebuilds `lists/*.md`. Chain list lives in this file under `workflow_run.workflows` — keep in sync when adding scrape workflows. |
|
||||
| `log_deleted.yml` | After check workflows | Exports `deleted_games.txt`. |
|
||||
| `deploy_webapp.yml` | Push to `main` touching `webapp/`, or manual | Builds → `docs/app/` → GitHub Pages. Requires repo Settings → Pages → Source = "GitHub Actions". |
|
||||
| `release_desktop.yml` | Tag `v*` push, or manual | Tauri build for Win + macOS aarch64 + macOS x86_64 (cross-compile) + Linux. Uploads `.dmg` / `.app.tar.gz` / `.pkg` (macOS), `.msi` / `.exe` (Windows), `.deb` / `.AppImage` (Linux). |
|
||||
@@ -89,7 +92,7 @@ Fine-grained PAT scoped to **only this repo** with:
|
||||
5. **Vite v8 `base`** only accepts `'./'`, an absolute URL, or an empty string. Anything else warns and silently breaks asset paths.
|
||||
6. **TS 6** deprecates `baseUrl`. Use `paths` alone.
|
||||
7. **lucide-react v1+** removed/renamed brand icons (`Github` → not available). Use `Library`, `Database`, etc. for repo icons.
|
||||
8. **`/repos/.../contents` `content` is base64 with newlines**; strip them before decoding (`atob(data.content.replace(/\n/g, ''))`).
|
||||
8. **`/repos/.../contents` `content` is base64 with newlines AND base64-decoded data is UTF-8.** Strip newlines first, but `atob` alone returns a Latin-1 binary string — multi-byte UTF-8 sequences become mojibake. Go through `TextDecoder('utf-8')` via the `base64ToUtf8` helper in `webapp/src/lib/github/encoding.ts`. The matching `utf8ToBase64` is correct as-is.
|
||||
9. **`createWorkflowDispatch` returns 204 with no run id.** To track the run, list runs filtered by `event=workflow_dispatch` and a dispatch timestamp from a few seconds before.
|
||||
10. **`docs/app/` is gitignored.** CI builds it. Don't commit build output.
|
||||
11. **PUT `/repos/.../contents/{path}` can never be "Verified".** GitHub authors that commit itself from the PAT; there is no `signature` field. All webapp writes must go through the Git Data API (blob → tree → commit → updateRef) to allow GPG signing. `contents.ts` `putFile` was removed for this reason — use `commitSingleFile` (wraps `atomicCommit`).
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Free Itch.io Games List
|
||||
|
||||
[](https://github.com/poli0981/free-games-itchio-list/)
|
||||
[](https://github.com/poli0981/free-games-itchio-list/)
|
||||
[](https://github.com/poli0981/free-games-itchio-list/stargazers)
|
||||
[](https://github.com/poli0981/free-games-itchio-list/network/members)
|
||||
[](https://github.com/poli0981/free-games-itchio-list/commits/main)
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
# Danh sách Game Itch.io Miễn Phí
|
||||
|
||||
[](https://github.com/poli0981/free-games-itchio-list/)
|
||||
[](https://github.com/poli0981/free-games-itchio-list/)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](README.md)
|
||||
|
||||
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
pip install --quiet beautifulsoup4 requests
|
||||
|
||||
# INPUT_URL is passed through from the workflow_dispatch input.
|
||||
python scripts/force_update.py
|
||||
|
||||
git config --global user.name "github-actions[bot]"
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add data_game/
|
||||
|
||||
if [[ -n "${INPUT_URL:-}" ]]; then
|
||||
msg="Force update ${INPUT_URL} [$(date +'%Y-%m-%d')]"
|
||||
else
|
||||
msg="Force update [$(date +'%Y-%m-%d')]"
|
||||
fi
|
||||
|
||||
git diff --staged --quiet && echo "No changes – skip commit" \
|
||||
|| (git commit -m "${msg}" && git push)
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
pip install --quiet beautifulsoup4 requests
|
||||
|
||||
python scripts/update_reviews.py
|
||||
|
||||
git config --global user.name "github-actions[bot]"
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add data_game/
|
||||
git diff --staged --quiet && echo "No changes – skip commit" \
|
||||
|| (git commit -m "Update reviews [$(date +'%Y-%m-%d')]" && git push)
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
pip install --quiet beautifulsoup4 requests
|
||||
|
||||
python scripts/update_status.py
|
||||
|
||||
git config --global user.name "github-actions[bot]"
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add data_game/
|
||||
git diff --staged --quiet && echo "No changes – skip commit" \
|
||||
|| (git commit -m "Update status [$(date +'%Y-%m-%d')]" && git push)
|
||||
@@ -0,0 +1,112 @@
|
||||
"""
|
||||
force_update.py — Emergency re-scrape of one or all existing games.
|
||||
|
||||
Manual `workflow_dispatch` only. Use when data has drifted or been corrupted
|
||||
(e.g. mojibake from an old read/write pipeline, missing fields after a scraper
|
||||
upgrade). Re-fetches the full itch.io page, replaces every scraper-owned field
|
||||
with fresh values, but ALWAYS preserves the three user-editable annotations:
|
||||
|
||||
safe_virus, notes, nsfw
|
||||
|
||||
(per CLAUDE.md "Editable vs read-only fields"). No "nuclear reset" mode —
|
||||
re-running this won't blow away your manual marks.
|
||||
|
||||
Inputs (env vars set by the workflow):
|
||||
INPUT_URL optional, single itch.io URL — re-scrape only this game.
|
||||
Empty / unset → re-scrape every game in data_game/.
|
||||
|
||||
Flow:
|
||||
1. Load all games from data_game/
|
||||
2. Pick targets (single URL or all)
|
||||
3. For each, scrape fresh; merge over old record, keeping the 3 preserved fields
|
||||
4. Network error / 404 → log warning, KEEP the old record unchanged
|
||||
(check_alive.py is the canonical path for pruning dead links)
|
||||
5. Save updated games (auto-rebalanced chunks)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from scraper import (
|
||||
create_session,
|
||||
scrape_game_info,
|
||||
polite_delay,
|
||||
batch_pause,
|
||||
should_batch_pause,
|
||||
)
|
||||
from data_store import load_all_games, save_all_games
|
||||
|
||||
# Fields the scraper does NOT own — never overwrite, regardless of scrape result.
|
||||
PRESERVE_FIELDS = ("safe_virus", "notes", "nsfw")
|
||||
|
||||
|
||||
def _normalize_url(u: str) -> str:
|
||||
return (u or "").strip().rstrip("/")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
games: list[dict] = load_all_games()
|
||||
if not games:
|
||||
print("No games found — nothing to update.")
|
||||
return
|
||||
|
||||
target_url = _normalize_url(os.environ.get("INPUT_URL", ""))
|
||||
|
||||
if target_url:
|
||||
targets = [g for g in games if _normalize_url(g.get("url", "")) == target_url]
|
||||
if not targets:
|
||||
print(f"URL not found in data_game/: {target_url}")
|
||||
sys.exit(1)
|
||||
print(f"Force-updating ONE game: {targets[0].get('name', '?')}")
|
||||
else:
|
||||
targets = games
|
||||
print(f"Force-updating ALL {len(targets)} games (preserving annotations).")
|
||||
|
||||
session = create_session()
|
||||
|
||||
updated = 0
|
||||
errors = 0
|
||||
not_found = 0
|
||||
total = len(targets)
|
||||
|
||||
for idx, game in enumerate(targets, start=1):
|
||||
url = game.get("url", "")
|
||||
name = game.get("name", "?")
|
||||
print(f"[{idx}/{total}] Re-scraping: {name}")
|
||||
|
||||
fresh = scrape_game_info(session, url)
|
||||
|
||||
if fresh is None:
|
||||
print(" → Network / parse error — keeping existing record.")
|
||||
errors += 1
|
||||
elif fresh.get("name", "").strip() in ("", "N/A"):
|
||||
# Empty name typically means the page returned 404 or a soft-removed game.
|
||||
print(" → Page returned no title — likely 404. Keeping existing record.")
|
||||
not_found += 1
|
||||
else:
|
||||
preserved = {k: game.get(k) for k in PRESERVE_FIELDS if k in game}
|
||||
game.clear()
|
||||
game.update(fresh)
|
||||
# is_free is a scrape-time flag, not part of the persisted schema.
|
||||
game.pop("is_free", None)
|
||||
for k, v in preserved.items():
|
||||
if v is not None:
|
||||
game[k] = v
|
||||
updated += 1
|
||||
|
||||
if should_batch_pause(idx):
|
||||
batch_pause()
|
||||
else:
|
||||
polite_delay()
|
||||
|
||||
save_all_games(games)
|
||||
|
||||
print(
|
||||
f"\nDone — {updated} re-scraped, {errors} errors, {not_found} unreachable "
|
||||
f"({total} targeted)."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,81 @@
|
||||
"""
|
||||
update_reviews.py — Refresh `rating` + `rating_count` on existing games.
|
||||
|
||||
Scheduled bi-weekly. Re-fetches each game's itch.io page, extracts only the two
|
||||
review-related fields, leaves everything else (including user annotations
|
||||
safe_virus / notes / nsfw) untouched.
|
||||
|
||||
Flow:
|
||||
1. Load all games from data_game/
|
||||
2. For each game, scrape fresh page → take rating + rating_count
|
||||
3. Network error / 404 → keep old values (don't blank them)
|
||||
4. Save updated games (auto-rebalanced chunks)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from scraper import (
|
||||
create_session,
|
||||
scrape_game_info,
|
||||
polite_delay,
|
||||
batch_pause,
|
||||
should_batch_pause,
|
||||
)
|
||||
from data_store import load_all_games, save_all_games
|
||||
|
||||
|
||||
def main() -> None:
|
||||
games: list[dict] = load_all_games()
|
||||
if not games:
|
||||
print("No games found — nothing to update.")
|
||||
return
|
||||
|
||||
session = create_session()
|
||||
|
||||
updated = 0
|
||||
skipped = 0
|
||||
errors = 0
|
||||
total = len(games)
|
||||
|
||||
for idx, game in enumerate(games, start=1):
|
||||
url = game.get("url", "")
|
||||
name = game.get("name", "?")
|
||||
print(f"[{idx}/{total}] Refreshing reviews: {name}")
|
||||
|
||||
fresh = scrape_game_info(session, url)
|
||||
|
||||
if fresh is None:
|
||||
# Network or parse error — keep old values
|
||||
print(" → Error, keeping old rating.")
|
||||
errors += 1
|
||||
else:
|
||||
new_rating = fresh.get("rating", "N/A")
|
||||
new_count = fresh.get("rating_count", "N/A")
|
||||
old_rating = game.get("rating", "N/A")
|
||||
old_count = game.get("rating_count", "N/A")
|
||||
if new_rating != old_rating or new_count != old_count:
|
||||
game["rating"] = new_rating
|
||||
game["rating_count"] = new_count
|
||||
updated += 1
|
||||
print(f" ✓ {old_rating} ({old_count}) → {new_rating} ({new_count})")
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
# Rate limiting
|
||||
if should_batch_pause(idx):
|
||||
batch_pause()
|
||||
else:
|
||||
polite_delay()
|
||||
|
||||
save_all_games(games)
|
||||
|
||||
print(
|
||||
f"\nDone — {updated} updated, {skipped} unchanged, {errors} errors "
|
||||
f"({total} games total)."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
"""
|
||||
update_status.py — Refresh `status` on existing games.
|
||||
|
||||
Scheduled monthly. Re-fetches each game's itch.io page, extracts only the
|
||||
`Status` field (e.g. Prototype → Released, In development → Canceled).
|
||||
|
||||
Flow:
|
||||
1. Load all games from data_game/
|
||||
2. For each game, scrape fresh page → take status
|
||||
3. Network error / 404 → keep old value
|
||||
4. Save updated games (auto-rebalanced chunks)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from scraper import (
|
||||
create_session,
|
||||
scrape_game_info,
|
||||
polite_delay,
|
||||
batch_pause,
|
||||
should_batch_pause,
|
||||
)
|
||||
from data_store import load_all_games, save_all_games
|
||||
|
||||
|
||||
def main() -> None:
|
||||
games: list[dict] = load_all_games()
|
||||
if not games:
|
||||
print("No games found — nothing to update.")
|
||||
return
|
||||
|
||||
session = create_session()
|
||||
|
||||
updated = 0
|
||||
skipped = 0
|
||||
errors = 0
|
||||
total = len(games)
|
||||
|
||||
for idx, game in enumerate(games, start=1):
|
||||
url = game.get("url", "")
|
||||
name = game.get("name", "?")
|
||||
print(f"[{idx}/{total}] Refreshing status: {name}")
|
||||
|
||||
fresh = scrape_game_info(session, url)
|
||||
|
||||
if fresh is None:
|
||||
print(" → Error, keeping old status.")
|
||||
errors += 1
|
||||
else:
|
||||
new_status = fresh.get("status", "N/A")
|
||||
old_status = game.get("status", "N/A")
|
||||
if new_status != old_status:
|
||||
game["status"] = new_status
|
||||
updated += 1
|
||||
print(f" ✓ {old_status} → {new_status}")
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
if should_batch_pause(idx):
|
||||
batch_pause()
|
||||
else:
|
||||
polite_delay()
|
||||
|
||||
save_all_games(games)
|
||||
|
||||
print(
|
||||
f"\nDone — {updated} updated, {skipped} unchanged, {errors} errors "
|
||||
f"({total} games total)."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -8,7 +8,7 @@ export interface ThirdParty {
|
||||
|
||||
export const APP = {
|
||||
name: 'Itch.io Free Games DB',
|
||||
version: '3.3.0',
|
||||
version: '3.4.0',
|
||||
repo: 'https://github.com/poli0981/free-games-itchio-list',
|
||||
license: 'MIT',
|
||||
buildDate: typeof __BUILD_DATE__ !== 'undefined' ? __BUILD_DATE__ : 'dev',
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Octokit } from '@octokit/rest'
|
||||
import { PATHS, REPO } from '../config'
|
||||
import { atomicCommit } from './git-data'
|
||||
import { rebalance } from './data-store'
|
||||
import { base64ToUtf8 } from './encoding'
|
||||
import type { Game } from '@/types/game'
|
||||
|
||||
interface FileWithSha {
|
||||
@@ -19,7 +20,7 @@ export async function readFileWithSha(octokit: Octokit, path: string): Promise<F
|
||||
if (Array.isArray(data) || data.type !== 'file' || !('content' in data)) {
|
||||
throw new Error(`${path} is not a file`)
|
||||
}
|
||||
const content = atob(data.content.replace(/\n/g, ''))
|
||||
const content = base64ToUtf8(data.content)
|
||||
return { content, sha: data.sha }
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* Base64 ↔ UTF-8 helpers for the GitHub Contents API.
|
||||
*
|
||||
* GitHub returns file content as a newline-padded base64 string. Decoding it with `atob` alone
|
||||
* yields a binary string in Latin-1 — multi-byte UTF-8 sequences (Vietnamese, Chinese, emoji…)
|
||||
* become mojibake when JSON.parse later treats those bytes as code points. Always go through
|
||||
* `TextDecoder('utf-8')` on the read side and `TextEncoder` on the write side.
|
||||
*/
|
||||
|
||||
export function base64ToUtf8(b64: string): string {
|
||||
const binary = atob(b64.replace(/\n/g, ''))
|
||||
const bytes = Uint8Array.from(binary, (c) => c.charCodeAt(0))
|
||||
return new TextDecoder('utf-8').decode(bytes)
|
||||
}
|
||||
|
||||
export function utf8ToBase64(s: string): string {
|
||||
const bytes = new TextEncoder().encode(s)
|
||||
let bin = ''
|
||||
for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i])
|
||||
return btoa(bin)
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { Octokit } from '@octokit/rest'
|
||||
import { PATHS, REPO } from '../config'
|
||||
import { rebalance } from './data-store'
|
||||
import { base64ToUtf8, utf8ToBase64 } from './encoding'
|
||||
import { buildCommitObject, currentTzOffsetMin, isoFromTs } from '../gpg/canonicalize'
|
||||
import { getSignerIfEnabled, type CommitSigner } from './signer'
|
||||
import type { Game } from '@/types/game'
|
||||
@@ -10,13 +11,6 @@ export interface FileChange {
|
||||
content: string | null
|
||||
}
|
||||
|
||||
function utf8ToBase64(s: string): string {
|
||||
const bytes = new TextEncoder().encode(s)
|
||||
let bin = ''
|
||||
for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i])
|
||||
return btoa(bin)
|
||||
}
|
||||
|
||||
export async function atomicCommit(
|
||||
octokit: Octokit,
|
||||
changes: FileChange[],
|
||||
@@ -205,7 +199,7 @@ export async function bulkDeleteGames(
|
||||
ref: REPO.branch,
|
||||
})
|
||||
if (!Array.isArray(data) && data.type === 'file' && 'content' in data) {
|
||||
const decoded = atob(data.content.replace(/\n/g, ''))
|
||||
const decoded = base64ToUtf8(data.content)
|
||||
deletedLog = JSON.parse(decoded || '[]')
|
||||
}
|
||||
} catch {
|
||||
|
||||
Reference in new issue
Block a user