386 lines
11 KiB
Bash
386 lines
11 KiB
Bash
#!/bin/bash
|
|
# Extract YouTube links from an Aegea (Egea) database and check them
|
|
# with y2dl / yt-dlp without downloading.
|
|
#
|
|
# Aegea stores note bodies as source text, not HTML:
|
|
# table ${DB_PREFIX}Notes.Text MEDIUMTEXT
|
|
# also ${DB_PREFIX}Notes.Summary
|
|
# also ${DB_PREFIX}Comments.Reply (author replies, full formatter)
|
|
# Prefix default is e2Blog. Alias type for notes is 'n'.
|
|
#
|
|
# Usage:
|
|
# DB_HOST=127.0.0.1 DB_NAME=blog DB_USER=blog DB_PASS=secret \
|
|
# ./egea-youtube-check.sh
|
|
#
|
|
# Optional:
|
|
# DB_PREFIX=e2Blog DB_SUBSET=1 YTDL=y2dl OUT_DIR=./yt-check \
|
|
# SOCKS5_PROXY=127.0.0.1:1080 ./egea-youtube-check.sh
|
|
#
|
|
# SOCKS5_PROXY is host:port, or a full socks5:// / socks5h:// URL.
|
|
# socks5h is used by default so DNS for YouTube also goes through the proxy.
|
|
|
|
set -euo pipefail
|
|
|
|
DB_HOST="${DB_HOST:-127.0.0.1}"
|
|
DB_PORT="${DB_PORT:-3306}"
|
|
DB_NAME="${DB_NAME:?set DB_NAME}"
|
|
DB_USER="${DB_USER:?set DB_USER}"
|
|
DB_PASS="${DB_PASS:-}"
|
|
DB_PREFIX="${DB_PREFIX:-e2Blog}"
|
|
DB_SUBSET="${DB_SUBSET:-1}"
|
|
# must be integer — it is interpolated into SQL
|
|
case "$DB_SUBSET" in
|
|
''|*[!0-9]*) echo "DB_SUBSET must be an integer, got: $DB_SUBSET" >&2; exit 1 ;;
|
|
esac
|
|
SITE_BASE="${SITE_BASE:-}"
|
|
YTDL="${YTDL:-y2dl}"
|
|
OUT_DIR="${OUT_DIR:-./yt-check}"
|
|
SLEEP_SEC="${SLEEP_SEC:-1}"
|
|
SOCKS5_PROXY="${SOCKS5_PROXY:-}"
|
|
|
|
proxy_url() {
|
|
local raw="$1"
|
|
[ -n "$raw" ] || return 0
|
|
case "$raw" in
|
|
socks5h://*|socks5://*|http://*|https://*)
|
|
printf "%s" "$raw"
|
|
;;
|
|
*)
|
|
printf "socks5h://%s" "$raw"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
PROXY_URL="$(proxy_url "$SOCKS5_PROXY")"
|
|
if [ -n "$PROXY_URL" ]; then
|
|
export ALL_PROXY="$PROXY_URL"
|
|
export all_proxy="$PROXY_URL"
|
|
export HTTPS_PROXY="$PROXY_URL"
|
|
export https_proxy="$PROXY_URL"
|
|
export HTTP_PROXY="$PROXY_URL"
|
|
export http_proxy="$PROXY_URL"
|
|
fi
|
|
|
|
YTDL_PROXY_ARGS=()
|
|
if [ -n "$PROXY_URL" ]; then
|
|
YTDL_PROXY_ARGS=(--proxy "$PROXY_URL")
|
|
fi
|
|
|
|
NOTES="${DB_PREFIX}Notes"
|
|
ALIASES="${DB_PREFIX}Aliases"
|
|
COMMENTS="${DB_PREFIX}Comments"
|
|
|
|
mkdir -p "$OUT_DIR"
|
|
|
|
mysql_q() {
|
|
# password via defaults file, not -p (avoids showing it in process list)
|
|
local cnf ec
|
|
cnf=$(mktemp)
|
|
chmod 600 "$cnf"
|
|
# escape for MySQL option file: backslash and double-quote
|
|
local u p h
|
|
u=$(printf '%s' "$DB_USER" | sed 's/\\/\\\\/g; s/"/\\"/g')
|
|
p=$(printf '%s' "$DB_PASS" | sed 's/\\/\\\\/g; s/"/\\"/g')
|
|
h=$(printf '%s' "$DB_HOST" | sed 's/\\/\\\\/g; s/"/\\"/g')
|
|
cat > "$cnf" <<EOF
|
|
[client]
|
|
user="${u}"
|
|
password="${p}"
|
|
host="${h}"
|
|
port=${DB_PORT}
|
|
EOF
|
|
# temporarily disable set -e so we always reach the cleanup
|
|
set +e
|
|
mysql \
|
|
--defaults-extra-file="$cnf" \
|
|
--batch --raw --skip-column-names \
|
|
"$DB_NAME" \
|
|
--default-character-set=utf8mb4 \
|
|
-e "$1"
|
|
ec=$?
|
|
set -e
|
|
rm -f "$cnf"
|
|
return $ec
|
|
}
|
|
|
|
mysql_q "
|
|
SELECT n.ID, n.IsPublished, n.IsVisible,
|
|
REPLACE(REPLACE(n.Title, '\t', ' '), '\n', ' '),
|
|
IFNULL(a.Alias, ''),
|
|
REPLACE(REPLACE(TO_BASE64(n.Text), '\n', ''), '\r', ''),
|
|
REPLACE(REPLACE(TO_BASE64(IFNULL(n.Summary, '')), '\n', ''), '\r', '')
|
|
FROM \`${NOTES}\` n
|
|
LEFT JOIN (
|
|
SELECT EntityID, Alias
|
|
FROM \`${ALIASES}\`
|
|
WHERE SubsetID = ${DB_SUBSET}
|
|
AND EntityType = 'n'
|
|
AND Stamp = (
|
|
SELECT MAX(Stamp)
|
|
FROM \`${ALIASES}\` a2
|
|
WHERE a2.SubsetID = ${DB_SUBSET}
|
|
AND a2.EntityType = 'n'
|
|
AND a2.EntityID = \`${ALIASES}\`.EntityID
|
|
)
|
|
) a ON a.EntityID = n.ID
|
|
WHERE n.SubsetID = ${DB_SUBSET}
|
|
AND (
|
|
n.Text LIKE '%youtu%'
|
|
OR n.Summary LIKE '%youtu%'
|
|
)
|
|
" > "$OUT_DIR/notes.tsv"
|
|
|
|
mysql_q "
|
|
SELECT c.ID, c.NoteID,
|
|
REPLACE(REPLACE(IFNULL(n.Title, ''), '\t', ' '), '\n', ' '),
|
|
IFNULL(a.Alias, ''),
|
|
REPLACE(REPLACE(TO_BASE64(c.Reply), '\n', ''), '\r', '')
|
|
FROM \`${COMMENTS}\` c
|
|
LEFT JOIN \`${NOTES}\` n ON n.ID = c.NoteID
|
|
LEFT JOIN (
|
|
SELECT EntityID, Alias
|
|
FROM \`${ALIASES}\`
|
|
WHERE SubsetID = ${DB_SUBSET}
|
|
AND EntityType = 'n'
|
|
AND Stamp = (
|
|
SELECT MAX(Stamp)
|
|
FROM \`${ALIASES}\` a2
|
|
WHERE a2.SubsetID = ${DB_SUBSET}
|
|
AND a2.EntityType = 'n'
|
|
AND a2.EntityID = \`${ALIASES}\`.EntityID
|
|
)
|
|
) a ON a.EntityID = c.NoteID
|
|
WHERE c.SubsetID = ${DB_SUBSET}
|
|
AND c.Reply LIKE '%youtu%'
|
|
" > "$OUT_DIR/replies.tsv"
|
|
|
|
python3 - "$OUT_DIR" "$SITE_BASE" << 'PY'
|
|
import re
|
|
import sys
|
|
import os
|
|
import base64
|
|
|
|
out_dir, site_base = sys.argv[1], sys.argv[2].rstrip("/")
|
|
site_base = site_base + "/" if site_base else ""
|
|
|
|
# Robust YouTube ID extractor.
|
|
# Handles:
|
|
# https://youtube.com/watch?v=ID
|
|
# youtube.com/watch?foo=bar&v=ID (no protocol, v not first)
|
|
# //youtu.be/ID
|
|
# embed/ID, shorts/ID, live/ID, v/ID
|
|
pat = re.compile(
|
|
r"(?:https?:)?//(?:www\.|m\.)?(?:"
|
|
r"youtube\.com/(?:"
|
|
r"watch\?(?:[^#\s]*&)?v=|"
|
|
r"embed/|shorts/|live/|v/"
|
|
r")|"
|
|
r"youtu\.be/"
|
|
r")([A-Za-z0-9_-]{11})"
|
|
r"|"
|
|
r"(?:^|[\s\"'(])(?:www\.|m\.)?(?:"
|
|
r"youtube\.com/(?:"
|
|
r"watch\?(?:[^#\s]*&)?v=|"
|
|
r"embed/|shorts/|live/|v/"
|
|
r")|"
|
|
r"youtu\.be/"
|
|
r")([A-Za-z0-9_-]{11})",
|
|
re.I,
|
|
)
|
|
|
|
def b64(s):
|
|
if not s:
|
|
return ""
|
|
try:
|
|
return base64.b64decode(s).decode("utf-8", errors="replace")
|
|
except Exception:
|
|
return s
|
|
|
|
def load_notes(path):
|
|
rows = []
|
|
if not os.path.exists(path) or os.path.getsize(path) == 0:
|
|
return rows
|
|
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
|
for line in f:
|
|
parts = line.rstrip("\n").split("\t")
|
|
if len(parts) < 7:
|
|
continue
|
|
note_id, published, visible, title, alias, text_b64, summary_b64 = parts[:7]
|
|
text = b64(text_b64)
|
|
summary = b64(summary_b64)
|
|
rows.append((note_id, published, visible, title, alias, text + "\n" + summary, "note"))
|
|
return rows
|
|
|
|
def load_replies(path):
|
|
rows = []
|
|
if not os.path.exists(path) or os.path.getsize(path) == 0:
|
|
return rows
|
|
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
|
for line in f:
|
|
parts = line.rstrip("\n").split("\t")
|
|
if len(parts) < 5:
|
|
continue
|
|
comment_id, note_id, title, alias, reply_b64 = parts[:5]
|
|
reply = b64(reply_b64)
|
|
rows.append((note_id, "", "", title, alias, reply, "reply:" + comment_id))
|
|
return rows
|
|
|
|
seen = {}
|
|
order = []
|
|
for note_id, published, visible, title, alias, blob, source in load_notes(os.path.join(out_dir, "notes.tsv")) + load_replies(os.path.join(out_dir, "replies.tsv")):
|
|
for m in pat.finditer(blob):
|
|
vid = m.group(1) or m.group(2)
|
|
if not vid:
|
|
continue
|
|
url = "https://www.youtube.com/watch?v=" + vid
|
|
key = vid
|
|
if key not in seen:
|
|
seen[key] = {
|
|
"id": vid,
|
|
"url": url,
|
|
"notes": [],
|
|
}
|
|
order.append(key)
|
|
# Aegea note URL is /all/<alias>/
|
|
if alias:
|
|
href = site_base + "all/" + alias.strip("/") + "/"
|
|
else:
|
|
href = ""
|
|
seen[key]["notes"].append({
|
|
"note_id": note_id,
|
|
"published": published,
|
|
"visible": visible,
|
|
"title": title,
|
|
"alias": alias,
|
|
"href": href,
|
|
"source": source,
|
|
"raw": m.group(0),
|
|
})
|
|
|
|
urls_path = os.path.join(out_dir, "urls.txt")
|
|
map_path = os.path.join(out_dir, "map.tsv")
|
|
with open(urls_path, "w", encoding="utf-8") as uf, open(map_path, "w", encoding="utf-8") as mf:
|
|
mf.write("video_id\turl\tnote_id\tpublished\tvisible\ttitle\talias\thref\tsource\traw\n")
|
|
for vid in order:
|
|
item = seen[vid]
|
|
uf.write(item["url"] + "\n")
|
|
for n in item["notes"]:
|
|
mf.write("\t".join([
|
|
vid,
|
|
item["url"],
|
|
n["note_id"],
|
|
n["published"],
|
|
n["visible"],
|
|
n["title"].replace("\t", " "),
|
|
n["alias"],
|
|
n["href"],
|
|
n["source"],
|
|
n["raw"],
|
|
]) + "\n")
|
|
|
|
print("unique_videos %d" % len(order))
|
|
print("urls %s" % urls_path)
|
|
print("map %s" % map_path)
|
|
PY
|
|
|
|
if [ -z "$PROXY_URL" ]; then
|
|
echo "SOCKS5_PROXY is empty, y2dl will go direct" >&2
|
|
else
|
|
echo "y2dl proxy $PROXY_URL" >&2
|
|
fi
|
|
|
|
if ! command -v "$YTDL" >/dev/null 2>&1; then
|
|
echo "extractor ok, checker missing: $YTDL" >&2
|
|
echo "set YTDL=yt-dlp or install y2dl, then rerun the check loop" >&2
|
|
exit 0
|
|
fi
|
|
|
|
REPORT="$OUT_DIR/report.tsv"
|
|
printf "video_id\tstatus\tdetail\tnotes\n" > "$REPORT"
|
|
|
|
# helper: collect unique note hrefs (or aliases/ids) for a video_id from map.tsv
|
|
note_links() {
|
|
local vid="$1"
|
|
awk -F '\t' -v v="$vid" '
|
|
NR>1 && $1==v {
|
|
link = $8
|
|
if (link == "") link = ($7 != "" ? $7 : "note:" $3)
|
|
seen[link] = 1
|
|
}
|
|
END {
|
|
first = 1
|
|
for (l in seen) {
|
|
if (!first) printf "; "
|
|
printf "%s", l
|
|
first = 0
|
|
}
|
|
}
|
|
' "$OUT_DIR/map.tsv"
|
|
}
|
|
|
|
# classify yt-dlp failure
|
|
# returns: unavailable | private | copyright | timeout | network | error
|
|
classify_error() {
|
|
local msg="$1"
|
|
local ec="$2"
|
|
[ "$ec" -eq 124 ] && { echo timeout; return; }
|
|
local low
|
|
low=$(printf '%s' "$msg" | tr '[:upper:]' '[:lower:]')
|
|
case "$low" in
|
|
*private*|*members*only*|*sign*in*to*confirm*) echo private ;;
|
|
*copyright*) echo copyright ;;
|
|
*unavailable*|*removed*|*deleted*|*terminated*|*does\ not\ exist*|*no\ longer\ available*) echo unavailable ;;
|
|
*timed*out*|*timeout*|*connection*|*proxy*|*network*|*resolve*|*refused*|*unreachable*) echo network ;;
|
|
*) echo error ;;
|
|
esac
|
|
}
|
|
|
|
total=$(grep -c . "$OUT_DIR/urls.txt" || true)
|
|
total=${total:-0}
|
|
n=0
|
|
|
|
echo "starting checks: $total videos, sleep ${SLEEP_SEC}s between them, timeout 45s per video"
|
|
echo "---"
|
|
|
|
while IFS= read -r url; do
|
|
[ -n "$url" ] || continue
|
|
n=$((n + 1))
|
|
vid="${url##*=}"
|
|
err_file="$OUT_DIR/err-${vid}.txt"
|
|
notes="$(note_links "$vid")"
|
|
printf '[%d/%d] %s ... ' "$n" "$total" "$vid" >&2
|
|
if detail="$(timeout 45 "$YTDL" "${YTDL_PROXY_ARGS[@]}" --skip-download --no-warnings --no-playlist -q --print "%(title)s" "$url" 2>"$err_file")"; then
|
|
title="$(printf '%s' "$detail" | head -n 1 | tr '\t' ' ')"
|
|
printf 'ok\t%s\n' "$title" >&2
|
|
printf '%s\tok\t%s\t%s\n' "$vid" "$title" "$notes" >> "$REPORT"
|
|
rm -f "$err_file"
|
|
else
|
|
ec=$?
|
|
raw="$(tr '\n' ' ' < "$err_file" 2>/dev/null | sed 's/[[:space:]]\+/ /g')"
|
|
[ -n "$raw" ] || raw="exit $ec"
|
|
status=$(classify_error "$raw" "$ec")
|
|
printf '%s\t%s\n' "$status" "$raw" >&2
|
|
printf '%s\t%s\t%s\t%s\n' "$vid" "$status" "$raw" "$notes" >> "$REPORT"
|
|
fi
|
|
sleep "$SLEEP_SEC"
|
|
done < "$OUT_DIR/urls.txt"
|
|
|
|
echo "---" >&2
|
|
echo "report $REPORT" >&2
|
|
|
|
# full report is already in report.tsv
|
|
# extract everything that is not ok into failed.tsv
|
|
FAILED="$OUT_DIR/failed.tsv"
|
|
printf "video_id\tstatus\tdetail\tnotes\n" > "$FAILED"
|
|
awk -F '\t' 'NR>1 && $2!="ok"' "$REPORT" >> "$FAILED"
|
|
|
|
failed_count=$(awk 'NR>1' "$FAILED" | wc -l | tr -d ' ')
|
|
ok_count=$((total - failed_count))
|
|
|
|
echo "summary: $ok_count ok, $failed_count failed (of $total)" >&2
|
|
echo "failed report: $FAILED" >&2
|
|
if [ "$failed_count" -gt 0 ]; then
|
|
echo "failed:" >&2
|
|
awk -F '\t' 'NR>1 {print $1 "\t" $2 "\t" $4 "\t" $3}' "$FAILED" >&2
|
|
fi
|