m9k/blog_data
1
0 Fork 0
Code Issues Actions Wiki Activity

Youtube checker

Browse Source
This commit is contained in:
latypov committed 2026-11-13 23:37:32 +07:00
1 parent 4bdeeebf68
commit 134d7c6259
7 files changed
+628
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

aegea_youtube_checler/.env.example
+30
View File
@@ -0,0 +1,30 @@
YTDLP_URL=https://github.com/yt-dlp/yt-dlp/releases/download/2026.10.26/yt-dlp
YTDLP_SHA256=e51e69e89489f199530f13d32a4342da3ddb2cf34842f364603a066db05afbf3
DENO_VERSION=2.9.8
# Proxy (socks5h:// recommended so DNS also goes through the proxy)
# Example: socks5h://user:pass@proxy.example:1080
# or just host:port (script will prefix socks5h://)
SOCKS5_PROXY=
ALL_PROXY=
HTTP_PROXY=
HTTPS_PROXY=
# Host directories
DOWNLOAD_DIR=./downloads
OUTPUT_DIR=./output
# Container user
UID=1000
GID=1000
# Egea / Aegea database (for the egea-check service)
DB_HOST=host.docker.internal
DB_PORT=3306
DB_NAME=blog
DB_USER=blog
DB_PASS=
DB_PREFIX=e2Blog
DB_SUBSET=1
SITE_BASE=https://blog.m9k.space/
SLEEP_SEC=1
aegea_youtube_checler/Dockerfile
+56
View File
@@ -0,0 +1,56 @@
FROM python:3.11-slim-bookworm
ARG YTDLP_URL=https://github.com/yt-dlp/yt-dlp/releases/download/2026.10.26/yt-dlp
ARG YTDLP_SHA256=e51e69e89489f199530f13d32a4342da3ddb2cf34842f364603a066db05afbf3
ARG DENO_VERSION=2.9.8
ENV DEBIAN_FRONTEND=noninteractive \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PATH="/usr/local/bin:${PATH}" \
YTDL=yt-dlp \
OUT_DIR=/output
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
curl \
ffmpeg \
unzip \
default-mysql-client \
&& rm -rf /var/lib/apt/lists/*
RUN set -eux; \
arch="$(dpkg --print-architecture)"; \
case "$arch" in \
amd64) deno_target="x86_64-unknown-linux-gnu" ;; \
arm64) deno_target="aarch64-unknown-linux-gnu" ;; \
*) echo "unsupported arch: $arch" >&2; exit 1 ;; \
esac; \
curl -fsSL "https://github.com/denoland/deno/releases/download/v${DENO_VERSION}/deno-${deno_target}.zip" \
-o "/tmp/deno-${deno_target}.zip"; \
curl -fsSL "https://github.com/denoland/deno/releases/download/v${DENO_VERSION}/deno-${deno_target}.zip.sha256sum" \
-o "/tmp/deno-${deno_target}.zip.sha256sum"; \
(cd /tmp && sha256sum -c "deno-${deno_target}.zip.sha256sum"); \
unzip -q "/tmp/deno-${deno_target}.zip" -d /usr/local/bin; \
chmod +x /usr/local/bin/deno; \
rm -f /tmp/deno-*; \
deno --version
RUN curl -fsSL "${YTDLP_URL}" -o /usr/local/bin/yt-dlp \
&& echo "${YTDLP_SHA256} /usr/local/bin/yt-dlp" | sha256sum -c - \
&& chmod +x /usr/local/bin/yt-dlp \
&& yt-dlp --version
COPY egea-youtube-check.sh /usr/local/bin/egea-youtube-check
RUN chmod +x /usr/local/bin/egea-youtube-check
RUN useradd --create-home --uid 1000 --shell /bin/bash app \
&& mkdir -p /downloads /output \
&& chown -R app:app /downloads /output
USER app
WORKDIR /downloads
ENTRYPOINT ["yt-dlp"]
CMD ["--help"]
aegea_youtube_checler/README.md
+92
View File
@@ -0,0 +1,92 @@
# yt-dlp 2026.10.26 + Egea YouTube checker
Минимальный образ: официальный zipimport-бинарник yt-dlp 2026.10.26, Python 3.11 (slim), ffmpeg/ffprobe, Deno 2.9.8, mysql-client и скрипт проверки ссылок из базы Эгеи.
Официальный репозиторий: https://github.com/yt-dlp/yt-dlp
## Файлы
```
yt-dlp-docker/
Dockerfile
docker-compose.yml
.env.example
egea-youtube-check.sh
downloads/ # монтируется в /downloads
output/ # монтируется в /output (результаты проверки)
```
## Параметры (.env)
| Переменная | По умолчанию | Назначение |
|----------------|---------------------------------------------------|------------|
| YTDLP_URL | https://github.com/yt-dlp/yt-dlp/releases/download/2026.10.26/yt-dlp | URL бинарника (build arg) |
| DENO_VERSION | 2.9.8 | Версия Deno (build arg) |
| SOCKS5_PROXY | пусто | SOCKS5 для yt-dlp (host:port или socks5h://…) |
| ALL_PROXY | = SOCKS5_PROXY | Альтернативный способ задать прокси |
| DOWNLOAD_DIR | ./downloads | Каталог загрузок на хосте |
| OUTPUT_DIR | ./output | Каталог отчётов проверки |
| UID / GID | 1000 | Пользователь контейнера |
| DB_HOST | host.docker.internal | Хост MariaDB/MySQL с базой Эгеи |
| DB_PORT | 3306 | Порт БД |
| DB_NAME | — | Имя базы |
| DB_USER | — | Пользователь |
| DB_PASS | — | Пароль |
| DB_PREFIX | e2Blog | Префикс таблиц |
| DB_SUBSET | 1 | SubsetID блога |
| SITE_BASE | пусто | Базовый URL сайта (для href в map.tsv / report). Эгея кладёт заметки в `/all/<alias>/`, скрипт сам добавит префикс. |
| SLEEP_SEC | 1 | Пауза между проверками |
## Использование
```bash
cd yt-dlp-docker
cp .env.example .env
# отредактировать .env (прокси, при необходимости параметры БД)
docker compose build
```
### Просто yt-dlp
```bash
docker compose run --rm yt-dlp --version
docker compose run --rm yt-dlp --skip-download "https://www.youtube.com/watch?v=jNQXAC9IVRw"
docker compose run --rm yt-dlp -o "%(title)s.%(ext)s" "URL"
```
### Проверка YouTube-ссылок из базы Эгеи
1. Заполнить в `.env` `DB_NAME`, `DB_USER`, `DB_PASS` и при необходимости `DB_HOST`.
2. Если база на хосте — убедиться, что MariaDB/MySQL слушает не только 127.0.0.1 (или использовать network_mode: host).
3. Запуск:
```bash
docker compose --profile check run --rm egea-check
```
Теперь скрипт печатает прогресс в реальном времени (в stderr):
```
[1/40] dQw4w9WgXcQ ... ok Never Gonna Give You Up
[2/40] xxxxxxxx ... unavailable ERROR: Video unavailable
[3/40] yyyyyyyy ... timeout timeout after 45s (proxy/network?)
```
Каждая строка появляется сразу после проверки ролика. Таймаут на один ролик — 45 секунд.
Результаты в `./output/`:
- `urls.txt` — уникальные ссылки
- `map.tsv` — соответствие video_id → посты/ответы
- `report.tsv` — полный отчёт. Статусы: `ok`, `unavailable`, `private`, `copyright`, `timeout`, `network`, `error`
- `failed.tsv` — всё, что не `ok` (с тем же статусом)
- `err-*.txt` — сырые ошибки для мёртвых роликов
В обоих отчётах колонка `notes` содержит href заметки (если задан `SITE_BASE`) или alias / `note:ID`. Если ролик в нескольких заметках — они перечислены через `; `.
Пример `failed.tsv`:
```
video_id status detail notes
dQw4w9... unavailable ERROR: Video unavailable https://blog.m9k.space/all/tema-vechera-7/
```
Deno включён по умолчанию, yt-dlp-ejs встроен в официальный бинарник.
aegea_youtube_checler/docker-compose.yml
+65
View File
@@ -0,0 +1,65 @@
services:
yt-dlp:
build:
context: .
args:
YTDLP_URL: ${YTDLP_URL:-https://github.com/yt-dlp/yt-dlp/releases/download/2026.10.26/yt-dlp}
YTDLP_SHA256: ${YTDLP_SHA256:-e51e69e89489f199530f13d32a4342da3ddb2cf34842f364603a066db05afbf3}
DENO_VERSION: ${DENO_VERSION:-2.9.8}
image: yt-dlp:2026.10.26
container_name: yt-dlp
user: "${UID:-1000}:${GID:-1000}"
working_dir: /downloads
volumes:
- ${DOWNLOAD_DIR:-./downloads}:/downloads
- ${OUTPUT_DIR:-./output}:/output
environment:
ALL_PROXY: ${ALL_PROXY:-${SOCKS5_PROXY:-}}
all_proxy: ${ALL_PROXY:-${SOCKS5_PROXY:-}}
HTTP_PROXY: ${HTTP_PROXY:-${ALL_PROXY:-${SOCKS5_PROXY:-}}}
http_proxy: ${HTTP_PROXY:-${ALL_PROXY:-${SOCKS5_PROXY:-}}}
HTTPS_PROXY: ${HTTPS_PROXY:-${ALL_PROXY:-${SOCKS5_PROXY:-}}}
https_proxy: ${HTTPS_PROXY:-${ALL_PROXY:-${SOCKS5_PROXY:-}}}
SOCKS5_PROXY: ${SOCKS5_PROXY:-}
extra_hosts:
- "host.docker.internal:host-gateway"
stdin_open: true
tty: true
entrypoint: ["yt-dlp"]
egea-check:
build:
context: .
args:
YTDLP_URL: ${YTDLP_URL:-https://github.com/yt-dlp/yt-dlp/releases/download/2026.10.26/yt-dlp}
YTDLP_SHA256: ${YTDLP_SHA256:-e51e69e89489f199530f13d32a4342da3ddb2cf34842f364603a066db05afbf3}
DENO_VERSION: ${DENO_VERSION:-2.9.8}
image: yt-dlp:2026.10.26
container_name: egea-check
user: "${UID:-1000}:${GID:-1000}"
volumes:
- ${OUTPUT_DIR:-./output}:/output
environment:
DB_HOST: ${DB_HOST:-host.docker.internal}
DB_PORT: ${DB_PORT:-3306}
DB_NAME: ${DB_NAME:-}
DB_USER: ${DB_USER:-}
DB_PASS: ${DB_PASS:-}
DB_PREFIX: ${DB_PREFIX:-e2Blog}
DB_SUBSET: ${DB_SUBSET:-1}
SITE_BASE: ${SITE_BASE:-}
YTDL: yt-dlp
OUT_DIR: /output
SLEEP_SEC: ${SLEEP_SEC:-1}
SOCKS5_PROXY: ${SOCKS5_PROXY:-}
ALL_PROXY: ${ALL_PROXY:-${SOCKS5_PROXY:-}}
all_proxy: ${ALL_PROXY:-${SOCKS5_PROXY:-}}
HTTP_PROXY: ${HTTP_PROXY:-${ALL_PROXY:-${SOCKS5_PROXY:-}}}
http_proxy: ${HTTP_PROXY:-${ALL_PROXY:-${SOCKS5_PROXY:-}}}
HTTPS_PROXY: ${HTTPS_PROXY:-${ALL_PROXY:-${SOCKS5_PROXY:-}}}
https_proxy: ${HTTPS_PROXY:-${ALL_PROXY:-${SOCKS5_PROXY:-}}}
extra_hosts:
- "host.docker.internal:host-gateway"
entrypoint: ["/usr/local/bin/egea-youtube-check"]
profiles:
- check
aegea_youtube_checler/downloads/.gitkeep
View File
Whitespace-only changes.
aegea_youtube_checler/egea-youtube-check.sh
+385
View File
@@ -0,0 +1,385 @@
#!/bin/bash
# Extract YouTube links from an Aegea (Egea) database and check them
# with y2dl / yt-dlp without downloading.
#
# Aegea stores note bodies as source text, not HTML:
# table ${DB_PREFIX}Notes.Text MEDIUMTEXT
# also ${DB_PREFIX}Notes.Summary
# also ${DB_PREFIX}Comments.Reply (author replies, full formatter)
# Prefix default is e2Blog. Alias type for notes is 'n'.
#
# Usage:
# DB_HOST=127.0.0.1 DB_NAME=blog DB_USER=blog DB_PASS=secret \
# ./egea-youtube-check.sh
#
# Optional:
# DB_PREFIX=e2Blog DB_SUBSET=1 YTDL=y2dl OUT_DIR=./yt-check \
# SOCKS5_PROXY=127.0.0.1:1080 ./egea-youtube-check.sh
#
# SOCKS5_PROXY is host:port, or a full socks5:// / socks5h:// URL.
# socks5h is used by default so DNS for YouTube also goes through the proxy.
set -euo pipefail
DB_HOST="${DB_HOST:-127.0.0.1}"
DB_PORT="${DB_PORT:-3306}"
DB_NAME="${DB_NAME:?set DB_NAME}"
DB_USER="${DB_USER:?set DB_USER}"
DB_PASS="${DB_PASS:-}"
DB_PREFIX="${DB_PREFIX:-e2Blog}"
DB_SUBSET="${DB_SUBSET:-1}"
# must be integer — it is interpolated into SQL
case "$DB_SUBSET" in
''|*[!0-9]*) echo "DB_SUBSET must be an integer, got: $DB_SUBSET" >&2; exit 1 ;;
esac
SITE_BASE="${SITE_BASE:-}"
YTDL="${YTDL:-y2dl}"
OUT_DIR="${OUT_DIR:-./yt-check}"
SLEEP_SEC="${SLEEP_SEC:-1}"
SOCKS5_PROXY="${SOCKS5_PROXY:-}"
proxy_url() {
local raw="$1"
[ -n "$raw" ] || return 0
case "$raw" in
socks5h://*|socks5://*|http://*|https://*)
printf "%s" "$raw"
;;
*)
printf "socks5h://%s" "$raw"
;;
esac
}
PROXY_URL="$(proxy_url "$SOCKS5_PROXY")"
if [ -n "$PROXY_URL" ]; then
export ALL_PROXY="$PROXY_URL"
export all_proxy="$PROXY_URL"
export HTTPS_PROXY="$PROXY_URL"
export https_proxy="$PROXY_URL"
export HTTP_PROXY="$PROXY_URL"
export http_proxy="$PROXY_URL"
fi
YTDL_PROXY_ARGS=()
if [ -n "$PROXY_URL" ]; then
YTDL_PROXY_ARGS=(--proxy "$PROXY_URL")
fi
NOTES="${DB_PREFIX}Notes"
ALIASES="${DB_PREFIX}Aliases"
COMMENTS="${DB_PREFIX}Comments"
mkdir -p "$OUT_DIR"
mysql_q() {
# password via defaults file, not -p (avoids showing it in process list)
local cnf ec
cnf=$(mktemp)
chmod 600 "$cnf"
# escape for MySQL option file: backslash and double-quote
local u p h
u=$(printf '%s' "$DB_USER" | sed 's/\\/\\\\/g; s/"/\\"/g')
p=$(printf '%s' "$DB_PASS" | sed 's/\\/\\\\/g; s/"/\\"/g')
h=$(printf '%s' "$DB_HOST" | sed 's/\\/\\\\/g; s/"/\\"/g')
cat > "$cnf" <<EOF
[client]
user="${u}"
password="${p}"
host="${h}"
port=${DB_PORT}
EOF
# temporarily disable set -e so we always reach the cleanup
set +e
mysql \
--defaults-extra-file="$cnf" \
--batch --raw --skip-column-names \
"$DB_NAME" \
--default-character-set=utf8mb4 \
-e "$1"
ec=$?
set -e
rm -f "$cnf"
return $ec
}
mysql_q "
SELECT n.ID, n.IsPublished, n.IsVisible,
REPLACE(REPLACE(n.Title, '\t', ' '), '\n', ' '),
IFNULL(a.Alias, ''),
REPLACE(REPLACE(TO_BASE64(n.Text), '\n', ''), '\r', ''),
REPLACE(REPLACE(TO_BASE64(IFNULL(n.Summary, '')), '\n', ''), '\r', '')
FROM \`${NOTES}\` n
LEFT JOIN (
SELECT EntityID, Alias
FROM \`${ALIASES}\`
WHERE SubsetID = ${DB_SUBSET}
AND EntityType = 'n'
AND Stamp = (
SELECT MAX(Stamp)
FROM \`${ALIASES}\` a2
WHERE a2.SubsetID = ${DB_SUBSET}
AND a2.EntityType = 'n'
AND a2.EntityID = \`${ALIASES}\`.EntityID
)
) a ON a.EntityID = n.ID
WHERE n.SubsetID = ${DB_SUBSET}
AND (
n.Text LIKE '%youtu%'
OR n.Summary LIKE '%youtu%'
)
" > "$OUT_DIR/notes.tsv"
mysql_q "
SELECT c.ID, c.NoteID,
REPLACE(REPLACE(IFNULL(n.Title, ''), '\t', ' '), '\n', ' '),
IFNULL(a.Alias, ''),
REPLACE(REPLACE(TO_BASE64(c.Reply), '\n', ''), '\r', '')
FROM \`${COMMENTS}\` c
LEFT JOIN \`${NOTES}\` n ON n.ID = c.NoteID
LEFT JOIN (
SELECT EntityID, Alias
FROM \`${ALIASES}\`
WHERE SubsetID = ${DB_SUBSET}
AND EntityType = 'n'
AND Stamp = (
SELECT MAX(Stamp)
FROM \`${ALIASES}\` a2
WHERE a2.SubsetID = ${DB_SUBSET}
AND a2.EntityType = 'n'
AND a2.EntityID = \`${ALIASES}\`.EntityID
)
) a ON a.EntityID = c.NoteID
WHERE c.SubsetID = ${DB_SUBSET}
AND c.Reply LIKE '%youtu%'
" > "$OUT_DIR/replies.tsv"
python3 - "$OUT_DIR" "$SITE_BASE" << 'PY'
import re
import sys
import os
import base64
out_dir, site_base = sys.argv[1], sys.argv[2].rstrip("/")
site_base = site_base + "/" if site_base else ""
# Robust YouTube ID extractor.
# Handles:
# https://youtube.com/watch?v=ID
# youtube.com/watch?foo=bar&v=ID (no protocol, v not first)
# //youtu.be/ID
# embed/ID, shorts/ID, live/ID, v/ID
pat = re.compile(
r"(?:https?:)?//(?:www\.|m\.)?(?:"
r"youtube\.com/(?:"
r"watch\?(?:[^#\s]*&)?v=|"
r"embed/|shorts/|live/|v/"
r")|"
r"youtu\.be/"
r")([A-Za-z0-9_-]{11})"
r"|"
r"(?:^|[\s\"'(])(?:www\.|m\.)?(?:"
r"youtube\.com/(?:"
r"watch\?(?:[^#\s]*&)?v=|"
r"embed/|shorts/|live/|v/"
r")|"
r"youtu\.be/"
r")([A-Za-z0-9_-]{11})",
re.I,
)
def b64(s):
if not s:
return ""
try:
return base64.b64decode(s).decode("utf-8", errors="replace")
except Exception:
return s
def load_notes(path):
rows = []
if not os.path.exists(path) or os.path.getsize(path) == 0:
return rows
with open(path, "r", encoding="utf-8", errors="replace") as f:
for line in f:
parts = line.rstrip("\n").split("\t")
if len(parts) < 7:
continue
note_id, published, visible, title, alias, text_b64, summary_b64 = parts[:7]
text = b64(text_b64)
summary = b64(summary_b64)
rows.append((note_id, published, visible, title, alias, text + "\n" + summary, "note"))
return rows
def load_replies(path):
rows = []
if not os.path.exists(path) or os.path.getsize(path) == 0:
return rows
with open(path, "r", encoding="utf-8", errors="replace") as f:
for line in f:
parts = line.rstrip("\n").split("\t")
if len(parts) < 5:
continue
comment_id, note_id, title, alias, reply_b64 = parts[:5]
reply = b64(reply_b64)
rows.append((note_id, "", "", title, alias, reply, "reply:" + comment_id))
return rows
seen = {}
order = []
for note_id, published, visible, title, alias, blob, source in load_notes(os.path.join(out_dir, "notes.tsv")) + load_replies(os.path.join(out_dir, "replies.tsv")):
for m in pat.finditer(blob):
vid = m.group(1) or m.group(2)
if not vid:
continue
url = "https://www.youtube.com/watch?v=" + vid
key = vid
if key not in seen:
seen[key] = {
"id": vid,
"url": url,
"notes": [],
}
order.append(key)
# Aegea note URL is /all/<alias>/
if alias:
href = site_base + "all/" + alias.strip("/") + "/"
else:
href = ""
seen[key]["notes"].append({
"note_id": note_id,
"published": published,
"visible": visible,
"title": title,
"alias": alias,
"href": href,
"source": source,
"raw": m.group(0),
})
urls_path = os.path.join(out_dir, "urls.txt")
map_path = os.path.join(out_dir, "map.tsv")
with open(urls_path, "w", encoding="utf-8") as uf, open(map_path, "w", encoding="utf-8") as mf:
mf.write("video_id\turl\tnote_id\tpublished\tvisible\ttitle\talias\thref\tsource\traw\n")
for vid in order:
item = seen[vid]
uf.write(item["url"] + "\n")
for n in item["notes"]:
mf.write("\t".join([
vid,
item["url"],
n["note_id"],
n["published"],
n["visible"],
n["title"].replace("\t", " "),
n["alias"],
n["href"],
n["source"],
n["raw"],
]) + "\n")
print("unique_videos %d" % len(order))
print("urls %s" % urls_path)
print("map %s" % map_path)
PY
if [ -z "$PROXY_URL" ]; then
echo "SOCKS5_PROXY is empty, y2dl will go direct" >&2
else
echo "y2dl proxy $PROXY_URL" >&2
fi
if ! command -v "$YTDL" >/dev/null 2>&1; then
echo "extractor ok, checker missing: $YTDL" >&2
echo "set YTDL=yt-dlp or install y2dl, then rerun the check loop" >&2
exit 0
fi
REPORT="$OUT_DIR/report.tsv"
printf "video_id\tstatus\tdetail\tnotes\n" > "$REPORT"
# helper: collect unique note hrefs (or aliases/ids) for a video_id from map.tsv
note_links() {
local vid="$1"
awk -F '\t' -v v="$vid" '
NR>1 && $1==v {
link = $8
if (link == "") link = ($7 != "" ? $7 : "note:" $3)
seen[link] = 1
}
END {
first = 1
for (l in seen) {
if (!first) printf "; "
printf "%s", l
first = 0
}
}
' "$OUT_DIR/map.tsv"
}
# classify yt-dlp failure
# returns: unavailable | private | copyright | timeout | network | error
classify_error() {
local msg="$1"
local ec="$2"
[ "$ec" -eq 124 ] && { echo timeout; return; }
local low
low=$(printf '%s' "$msg" | tr '[:upper:]' '[:lower:]')
case "$low" in
*private*|*members*only*|*sign*in*to*confirm*) echo private ;;
*copyright*) echo copyright ;;
*unavailable*|*removed*|*deleted*|*terminated*|*does\ not\ exist*|*no\ longer\ available*) echo unavailable ;;
*timed*out*|*timeout*|*connection*|*proxy*|*network*|*resolve*|*refused*|*unreachable*) echo network ;;
*) echo error ;;
esac
}
total=$(grep -c . "$OUT_DIR/urls.txt" || true)
total=${total:-0}
n=0
echo "starting checks: $total videos, sleep ${SLEEP_SEC}s between them, timeout 45s per video"
echo "---"
while IFS= read -r url; do
[ -n "$url" ] || continue
n=$((n + 1))
vid="${url##*=}"
err_file="$OUT_DIR/err-${vid}.txt"
notes="$(note_links "$vid")"
printf '[%d/%d] %s ... ' "$n" "$total" "$vid" >&2
if detail="$(timeout 45 "$YTDL" "${YTDL_PROXY_ARGS[@]}" --skip-download --no-warnings --no-playlist -q --print "%(title)s" "$url" 2>"$err_file")"; then
title="$(printf '%s' "$detail" | head -n 1 | tr '\t' ' ')"
printf 'ok\t%s\n' "$title" >&2
printf '%s\tok\t%s\t%s\n' "$vid" "$title" "$notes" >> "$REPORT"
rm -f "$err_file"
else
ec=$?
raw="$(tr '\n' ' ' < "$err_file" 2>/dev/null | sed 's/[[:space:]]\+/ /g')"
[ -n "$raw" ] || raw="exit $ec"
status=$(classify_error "$raw" "$ec")
printf '%s\t%s\n' "$status" "$raw" >&2
printf '%s\t%s\t%s\t%s\n' "$vid" "$status" "$raw" "$notes" >> "$REPORT"
fi
sleep "$SLEEP_SEC"
done < "$OUT_DIR/urls.txt"
echo "---" >&2
echo "report $REPORT" >&2
# full report is already in report.tsv
# extract everything that is not ok into failed.tsv
FAILED="$OUT_DIR/failed.tsv"
printf "video_id\tstatus\tdetail\tnotes\n" > "$FAILED"
awk -F '\t' 'NR>1 && $2!="ok"' "$REPORT" >> "$FAILED"
failed_count=$(awk 'NR>1' "$FAILED" | wc -l | tr -d ' ')
ok_count=$((total - failed_count))
echo "summary: $ok_count ok, $failed_count failed (of $total)" >&2
echo "failed report: $FAILED" >&2
if [ "$failed_count" -gt 0 ]; then
echo "failed:" >&2
awk -F '\t' 'NR>1 {print $1 "\t" $2 "\t" $4 "\t" $3}' "$FAILED" >&2
fi
aegea_youtube_checler/output/.gitkeep
View File
Whitespace-only changes.