#!/bin/bash # Extract YouTube links from an Aegea (Egea) database and check them # with y2dl / yt-dlp without downloading. # # Aegea stores note bodies as source text, not HTML: # table ${DB_PREFIX}Notes.Text MEDIUMTEXT # also ${DB_PREFIX}Notes.Summary # also ${DB_PREFIX}Comments.Reply (author replies, full formatter) # Prefix default is e2Blog. Alias type for notes is 'n'. # # Usage: # DB_HOST=127.0.0.1 DB_NAME=blog DB_USER=blog DB_PASS=secret \ # ./egea-youtube-check.sh # # Optional: # DB_PREFIX=e2Blog DB_SUBSET=1 YTDL=y2dl OUT_DIR=./yt-check \ # SOCKS5_PROXY=127.0.0.1:1080 ./egea-youtube-check.sh # # SOCKS5_PROXY is host:port, or a full socks5:// / socks5h:// URL. # socks5h is used by default so DNS for YouTube also goes through the proxy. set -euo pipefail DB_HOST="${DB_HOST:-127.0.0.1}" DB_PORT="${DB_PORT:-3306}" DB_NAME="${DB_NAME:?set DB_NAME}" DB_USER="${DB_USER:?set DB_USER}" DB_PASS="${DB_PASS:-}" DB_PREFIX="${DB_PREFIX:-e2Blog}" DB_SUBSET="${DB_SUBSET:-1}" # must be integer — it is interpolated into SQL case "$DB_SUBSET" in ''|*[!0-9]*) echo "DB_SUBSET must be an integer, got: $DB_SUBSET" >&2; exit 1 ;; esac SITE_BASE="${SITE_BASE:-}" YTDL="${YTDL:-y2dl}" OUT_DIR="${OUT_DIR:-./yt-check}" SLEEP_SEC="${SLEEP_SEC:-1}" SOCKS5_PROXY="${SOCKS5_PROXY:-}" proxy_url() { local raw="$1" [ -n "$raw" ] || return 0 case "$raw" in socks5h://*|socks5://*|http://*|https://*) printf "%s" "$raw" ;; *) printf "socks5h://%s" "$raw" ;; esac } PROXY_URL="$(proxy_url "$SOCKS5_PROXY")" if [ -n "$PROXY_URL" ]; then export ALL_PROXY="$PROXY_URL" export all_proxy="$PROXY_URL" export HTTPS_PROXY="$PROXY_URL" export https_proxy="$PROXY_URL" export HTTP_PROXY="$PROXY_URL" export http_proxy="$PROXY_URL" fi YTDL_PROXY_ARGS=() if [ -n "$PROXY_URL" ]; then YTDL_PROXY_ARGS=(--proxy "$PROXY_URL") fi NOTES="${DB_PREFIX}Notes" ALIASES="${DB_PREFIX}Aliases" COMMENTS="${DB_PREFIX}Comments" mkdir -p "$OUT_DIR" mysql_q() { # password via defaults file, not -p (avoids showing it in process list) local cnf ec cnf=$(mktemp) chmod 600 "$cnf" # escape for MySQL option file: backslash and double-quote local u p h u=$(printf '%s' "$DB_USER" | sed 's/\\/\\\\/g; s/"/\\"/g') p=$(printf '%s' "$DB_PASS" | sed 's/\\/\\\\/g; s/"/\\"/g') h=$(printf '%s' "$DB_HOST" | sed 's/\\/\\\\/g; s/"/\\"/g') cat > "$cnf" < "$OUT_DIR/notes.tsv" mysql_q " SELECT c.ID, c.NoteID, REPLACE(REPLACE(IFNULL(n.Title, ''), '\t', ' '), '\n', ' '), IFNULL(a.Alias, ''), REPLACE(REPLACE(TO_BASE64(c.Reply), '\n', ''), '\r', '') FROM \`${COMMENTS}\` c LEFT JOIN \`${NOTES}\` n ON n.ID = c.NoteID LEFT JOIN ( SELECT EntityID, Alias FROM \`${ALIASES}\` WHERE SubsetID = ${DB_SUBSET} AND EntityType = 'n' AND Stamp = ( SELECT MAX(Stamp) FROM \`${ALIASES}\` a2 WHERE a2.SubsetID = ${DB_SUBSET} AND a2.EntityType = 'n' AND a2.EntityID = \`${ALIASES}\`.EntityID ) ) a ON a.EntityID = c.NoteID WHERE c.SubsetID = ${DB_SUBSET} AND c.Reply LIKE '%youtu%' " > "$OUT_DIR/replies.tsv" python3 - "$OUT_DIR" "$SITE_BASE" << 'PY' import re import sys import os import base64 out_dir, site_base = sys.argv[1], sys.argv[2].rstrip("/") site_base = site_base + "/" if site_base else "" # Robust YouTube ID extractor. # Handles: # https://youtube.com/watch?v=ID # youtube.com/watch?foo=bar&v=ID (no protocol, v not first) # //youtu.be/ID # embed/ID, shorts/ID, live/ID, v/ID pat = re.compile( r"(?:https?:)?//(?:www\.|m\.)?(?:" r"youtube\.com/(?:" r"watch\?(?:[^#\s]*&)?v=|" r"embed/|shorts/|live/|v/" r")|" r"youtu\.be/" r")([A-Za-z0-9_-]{11})" r"|" r"(?:^|[\s\"'(])(?:www\.|m\.)?(?:" r"youtube\.com/(?:" r"watch\?(?:[^#\s]*&)?v=|" r"embed/|shorts/|live/|v/" r")|" r"youtu\.be/" r")([A-Za-z0-9_-]{11})", re.I, ) def b64(s): if not s: return "" try: return base64.b64decode(s).decode("utf-8", errors="replace") except Exception: return s def load_notes(path): rows = [] if not os.path.exists(path) or os.path.getsize(path) == 0: return rows with open(path, "r", encoding="utf-8", errors="replace") as f: for line in f: parts = line.rstrip("\n").split("\t") if len(parts) < 7: continue note_id, published, visible, title, alias, text_b64, summary_b64 = parts[:7] text = b64(text_b64) summary = b64(summary_b64) rows.append((note_id, published, visible, title, alias, text + "\n" + summary, "note")) return rows def load_replies(path): rows = [] if not os.path.exists(path) or os.path.getsize(path) == 0: return rows with open(path, "r", encoding="utf-8", errors="replace") as f: for line in f: parts = line.rstrip("\n").split("\t") if len(parts) < 5: continue comment_id, note_id, title, alias, reply_b64 = parts[:5] reply = b64(reply_b64) rows.append((note_id, "", "", title, alias, reply, "reply:" + comment_id)) return rows seen = {} order = [] for note_id, published, visible, title, alias, blob, source in load_notes(os.path.join(out_dir, "notes.tsv")) + load_replies(os.path.join(out_dir, "replies.tsv")): for m in pat.finditer(blob): vid = m.group(1) or m.group(2) if not vid: continue url = "https://www.youtube.com/watch?v=" + vid key = vid if key not in seen: seen[key] = { "id": vid, "url": url, "notes": [], } order.append(key) # Aegea note URL is /all// if alias: href = site_base + "all/" + alias.strip("/") + "/" else: href = "" seen[key]["notes"].append({ "note_id": note_id, "published": published, "visible": visible, "title": title, "alias": alias, "href": href, "source": source, "raw": m.group(0), }) urls_path = os.path.join(out_dir, "urls.txt") map_path = os.path.join(out_dir, "map.tsv") with open(urls_path, "w", encoding="utf-8") as uf, open(map_path, "w", encoding="utf-8") as mf: mf.write("video_id\turl\tnote_id\tpublished\tvisible\ttitle\talias\thref\tsource\traw\n") for vid in order: item = seen[vid] uf.write(item["url"] + "\n") for n in item["notes"]: mf.write("\t".join([ vid, item["url"], n["note_id"], n["published"], n["visible"], n["title"].replace("\t", " "), n["alias"], n["href"], n["source"], n["raw"], ]) + "\n") print("unique_videos %d" % len(order)) print("urls %s" % urls_path) print("map %s" % map_path) PY if [ -z "$PROXY_URL" ]; then echo "SOCKS5_PROXY is empty, y2dl will go direct" >&2 else echo "y2dl proxy $PROXY_URL" >&2 fi if ! command -v "$YTDL" >/dev/null 2>&1; then echo "extractor ok, checker missing: $YTDL" >&2 echo "set YTDL=yt-dlp or install y2dl, then rerun the check loop" >&2 exit 0 fi REPORT="$OUT_DIR/report.tsv" printf "video_id\tstatus\tdetail\tnotes\n" > "$REPORT" # helper: collect unique note hrefs (or aliases/ids) for a video_id from map.tsv note_links() { local vid="$1" awk -F '\t' -v v="$vid" ' NR>1 && $1==v { link = $8 if (link == "") link = ($7 != "" ? $7 : "note:" $3) seen[link] = 1 } END { first = 1 for (l in seen) { if (!first) printf "; " printf "%s", l first = 0 } } ' "$OUT_DIR/map.tsv" } # classify yt-dlp failure # returns: unavailable | private | copyright | timeout | network | error classify_error() { local msg="$1" local ec="$2" [ "$ec" -eq 124 ] && { echo timeout; return; } local low low=$(printf '%s' "$msg" | tr '[:upper:]' '[:lower:]') case "$low" in *private*|*members*only*|*sign*in*to*confirm*) echo private ;; *copyright*) echo copyright ;; *unavailable*|*removed*|*deleted*|*terminated*|*does\ not\ exist*|*no\ longer\ available*) echo unavailable ;; *timed*out*|*timeout*|*connection*|*proxy*|*network*|*resolve*|*refused*|*unreachable*) echo network ;; *) echo error ;; esac } total=$(grep -c . "$OUT_DIR/urls.txt" || true) total=${total:-0} n=0 echo "starting checks: $total videos, sleep ${SLEEP_SEC}s between them, timeout 45s per video" echo "---" while IFS= read -r url; do [ -n "$url" ] || continue n=$((n + 1)) vid="${url##*=}" err_file="$OUT_DIR/err-${vid}.txt" notes="$(note_links "$vid")" printf '[%d/%d] %s ... ' "$n" "$total" "$vid" >&2 if detail="$(timeout 45 "$YTDL" "${YTDL_PROXY_ARGS[@]}" --skip-download --no-warnings --no-playlist -q --print "%(title)s" "$url" 2>"$err_file")"; then title="$(printf '%s' "$detail" | head -n 1 | tr '\t' ' ')" printf 'ok\t%s\n' "$title" >&2 printf '%s\tok\t%s\t%s\n' "$vid" "$title" "$notes" >> "$REPORT" rm -f "$err_file" else ec=$? raw="$(tr '\n' ' ' < "$err_file" 2>/dev/null | sed 's/[[:space:]]\+/ /g')" [ -n "$raw" ] || raw="exit $ec" status=$(classify_error "$raw" "$ec") printf '%s\t%s\n' "$status" "$raw" >&2 printf '%s\t%s\t%s\t%s\n' "$vid" "$status" "$raw" "$notes" >> "$REPORT" fi sleep "$SLEEP_SEC" done < "$OUT_DIR/urls.txt" echo "---" >&2 echo "report $REPORT" >&2 # full report is already in report.tsv # extract everything that is not ok into failed.tsv FAILED="$OUT_DIR/failed.tsv" printf "video_id\tstatus\tdetail\tnotes\n" > "$FAILED" awk -F '\t' 'NR>1 && $2!="ok"' "$REPORT" >> "$FAILED" failed_count=$(awk 'NR>1' "$FAILED" | wc -l | tr -d ' ') ok_count=$((total - failed_count)) echo "summary: $ok_count ok, $failed_count failed (of $total)" >&2 echo "failed report: $FAILED" >&2 if [ "$failed_count" -gt 0 ]; then echo "failed:" >&2 awk -F '\t' 'NR>1 {print $1 "\t" $2 "\t" $4 "\t" $3}' "$FAILED" >&2 fi