From 4e7b8e3e48fa0521324dcf170fee253241ce67c9 Mon Sep 17 00:00:00 2001 From: Release Monitor Date: Sun, 11 Oct 2026 14:29:42 +0000 Subject: [PATCH] Improve repository config, changelog notifications and API resilience --- .env.example | 4 +++ README.md | 9 +++++++ monitor.sh | 75 +++++++++++++++++++++++++++++++++++++++------------- repos.json | 7 +++-- 4 files changed, 73 insertions(+), 22 deletions(-) diff --git a/.env.example b/.env.example index 3adddef..730ce59 100644 --- a/.env.example +++ b/.env.example @@ -8,3 +8,7 @@ GITHUB_MAX_PAGES=3 REPOS_FILE=repos.json STATE_FILE=state.json INCLUDE_PRERELEASES=false + +TELEGRAM_INCLUDE_CHANGELOG=true +TELEGRAM_CHANGELOG_MAX_LENGTH=1800 +STATE_MAX_IDS=500 diff --git a/README.md b/README.md index 932133f..7bc21f6 100644 --- a/README.md +++ b/README.md @@ -65,3 +65,12 @@ through command-line arguments. The state file is read directly by `jq`. This prevents `Argument list too long` for repositories with large release notes. Run `bash -n monitor.sh` and `./monitor.sh --dry-run --verbose` to verify. + +### Release notes and retention + +`repos.json` uses `owner/repo` names (legacy full GitHub URLs remain accepted). +`TELEGRAM_INCLUDE_CHANGELOG=true` includes a shortened release body in Telegram messages. +`TELEGRAM_CHANGELOG_MAX_LENGTH=1800` limits the changelog; Telegram messages are capped at 4096 characters. +`STATE_MAX_IDS=500` bounds stored release IDs per repository. Keep this larger than the number of releases that may appear in the fetched pages; old releases returning after pruning may otherwise be notified again. + +GitHub HTTP 403/429 responses log rate-limit headers and fail without changing the repository state. Mid-pagination errors also fail the repository without partially processing it. Tags without GitHub releases are not tracked. diff --git a/monitor.sh b/monitor.sh index 7e30030..f3eddd1 100755 --- a/monitor.sh +++ b/monitor.sh @@ -57,6 +57,9 @@ set +a : "${REPOS_FILE:=repos.json}" : "${STATE_FILE:=state.json}" : "${INCLUDE_PRERELEASES:=false}" +: "${TELEGRAM_INCLUDE_CHANGELOG:=true}" +: "${TELEGRAM_CHANGELOG_MAX_LENGTH:=1800}" +: "${STATE_MAX_IDS:=500}" if [[ "$CLI_PRERELEASES" == true ]]; then INCLUDE_PRERELEASES=true; fi resolve_path() { if [[ "$1" == /* ]]; then printf '%s\n' "$1"; else printf '%s/%s\n' "$SCRIPT_DIR" "$1"; fi; } REPOS_FILE="$(resolve_path "$REPOS_FILE")" @@ -67,12 +70,18 @@ for cmd in curl jq flock mktemp; do command -v "$cmd" >/dev/null 2>&1 || die "Mi (( GITHUB_PER_PAGE <= 100 )) || die "GITHUB_PER_PAGE must be <= 100" [[ "$GITHUB_MAX_PAGES" =~ ^[1-9][0-9]*$ ]] || die "Invalid GITHUB_MAX_PAGES" [[ "$INCLUDE_PRERELEASES" == true || "$INCLUDE_PRERELEASES" == false ]] || die "Invalid INCLUDE_PRERELEASES" +for var in TELEGRAM_CHANGELOG_MAX_LENGTH STATE_MAX_IDS; do + [[ "${!var}" =~ ^[1-9][0-9]*$ ]] || die "Invalid $var" +done +[[ "$TELEGRAM_INCLUDE_CHANGELOG" == true || "$TELEGRAM_INCLUDE_CHANGELOG" == false ]] || die 'Invalid TELEGRAM_INCLUDE_CHANGELOG' if [[ "$TEST_TELEGRAM" == true || ( "$INIT" == false && "$DRY_RUN" == false ) ]]; then [[ -n "$TELEGRAM_BOT_TOKEN" && -n "$TELEGRAM_CHAT_ID" ]] || die "Telegram credentials are missing" fi umask 077 CURL_OPTIONS=(--silent --show-error --fail --location --connect-timeout 10 --max-time "$REQUEST_TIMEOUT") if [[ -n "$SOCKS5_PROXY" ]]; then CURL_OPTIONS+=(--proxy "$SOCKS5_PROXY"); fi +GITHUB_CURL_OPTIONS=(--silent --show-error --location --connect-timeout 10 --max-time "$REQUEST_TIMEOUT") +if [[ -n "$SOCKS5_PROXY" ]]; then GITHUB_CURL_OPTIONS+=(--proxy "$SOCKS5_PROXY"); fi GITHUB_HEADERS=(-H 'Accept: application/vnd.github+json' -H 'X-GitHub-Api-Version: 2022-11-28' -H 'User-Agent: github-release-monitor') if [[ -n "$GITHUB_TOKEN" ]]; then GITHUB_HEADERS+=(-H "Authorization: Bearer ${GITHUB_TOKEN}"); fi debug() { if [[ "$VERBOSE" == true ]]; then log DEBUG "$1"; fi; } @@ -89,33 +98,47 @@ send_telegram() { test_telegram() { local message message="$(printf 'GitHub Release Monitor: Test notification\nHost: %s\nTime: %s\nStatus: Telegram connection successful' "$(hostname)" "$(date '+%Y-%m-%d %H:%M:%S %Z')")" + if [[ "$TELEGRAM_INCLUDE_CHANGELOG" == true ]]; then + message+=$'\n\nChangelog:\nTest changelog rendering enabled.' + fi log INFO 'Sending test Telegram notification' if send_telegram "$message"; then log INFO 'Test notification sent'; else log ERROR 'Test notification failed'; return 1; fi } fetch_releases() { - local repo="$1" page count response_file releases_file + local repo="$1" page count response_file releases_file headers_file http_code retry_after remaining reset response_file="$(mktemp)" || return 1 releases_file="$(mktemp)" || { rm -f "$response_file"; return 1; } + headers_file="$(mktemp)" || { rm -f "$response_file" "$releases_file"; return 1; } for ((page=1; page<=GITHUB_MAX_PAGES; page++)); do debug "Fetching $repo page=$page" - if ! curl "${CURL_OPTIONS[@]}" "${GITHUB_HEADERS[@]}" \ + if ! http_code="$(curl "${GITHUB_CURL_OPTIONS[@]}" "${GITHUB_HEADERS[@]}" \ + --dump-header "$headers_file" --write-out '%{http_code}' \ --output "$response_file" \ - "${GITHUB_API}/repos/${repo}/releases?per_page=${GITHUB_PER_PAGE}&page=${page}"; then - rm -f "$response_file" "$releases_file" + "${GITHUB_API}/repos/${repo}/releases?per_page=${GITHUB_PER_PAGE}&page=${page}")"; then + rm -f "$response_file" "$releases_file" "$headers_file" + return 1 + fi + if [[ "$http_code" != 200 ]]; then + retry_after="$(awk 'BEGIN{IGNORECASE=1} tolower($1)=="retry-after:" {gsub("\r", "", $2); print $2}' "$headers_file" | tail -n 1)" + remaining="$(awk 'tolower($1)=="x-ratelimit-remaining:" {gsub("\r", "", $2); print $2}' "$headers_file" | tail -n 1)" + reset="$(awk 'tolower($1)=="x-ratelimit-reset:" {gsub("\r", "", $2); print $2}' "$headers_file" | tail -n 1)" + log ERROR "GitHub HTTP $http_code for $repo; rate_remaining=${remaining:-unknown}; rate_reset=${reset:-unknown}; retry_after=${retry_after:-unknown}" + rm -f "$response_file" "$releases_file" "$headers_file" return 1 fi if ! count="$(jq -er 'if type == "array" then length else error("Invalid releases response") end' "$response_file")"; then - rm -f "$response_file" "$releases_file" + rm -f "$response_file" "$releases_file" "$headers_file" return 1 fi if ! jq -c --argjson prereleases "$INCLUDE_PRERELEASES" ' .[] | select(.draft == false) | select($prereleases or (.prerelease == false)) | {id: (.id | tostring), tag: .tag_name, name: .name, - url: .html_url, published: .published_at, prerelease: .prerelease} + url: .html_url, published: .published_at, prerelease: .prerelease, + body: (.body // "")} ' "$response_file" >> "$releases_file"; then - rm -f "$response_file" "$releases_file" + rm -f "$response_file" "$releases_file" "$headers_file" return 1 fi if (( count < GITHUB_PER_PAGE )); then break; fi @@ -123,22 +146,22 @@ fetch_releases() { local status=0 jq -sc '.' "$releases_file" || status=$? - rm -f "$response_file" "$releases_file" + rm -f "$response_file" "$releases_file" "$headers_file" return "$status" } write_state() { local repo="$1" ids="$2" mode="$3" temp temp="$(mktemp "${STATE_FILE}.tmp.XXXXXX")" || return 1 - if ! jq --arg repo "$repo" --arg mode "$mode" --slurpfile ids /dev/stdin ' + if ! jq --arg repo "$repo" --arg mode "$mode" --argjson limit "$STATE_MAX_IDS" --slurpfile ids /dev/stdin ' ($ids[0] // []) as $new_ids | - if $mode == "replace" then .[$repo] = $new_ids - else .[$repo] = ((.[$repo] // []) + $new_ids | unique) end + if $mode == "replace" then .[$repo] = ($new_ids | unique | .[-$limit:]) + else .[$repo] = ((.[$repo] // []) + $new_ids | unique | .[-$limit:]) end ' "$STATE_FILE" <<< "$ids" > "$temp"; then rm -f "$temp"; return 1; fi if ! mv -f "$temp" "$STATE_FILE"; then rm -f "$temp"; return 1; fi } format_message() { - local repo="$1" release="$2" tag name url published kind + local repo="$1" release="$2" tag name url published kind body message allowance tag="$(jq -r '.tag // "unknown"' <<< "$release")" name="$(jq -r '.name // empty' <<< "$release")" url="$(jq -r '.url // empty' <<< "$release")" @@ -146,8 +169,20 @@ format_message() { [[ -n "$name" ]] || name="$tag" kind=Release if [[ "$(jq -r '.prerelease' <<< "$release")" == true ]]; then kind=Prerelease; fi - printf 'GitHub: New %s\nRepository: %s\nVersion: %s\nName: %s\nPublished: %s\nURL: %s' \ - "$kind" "$repo" "$tag" "$name" "$published" "$url" + message="$(printf 'GitHub: New %s\nRepository: %s\nVersion: %s\nName: %s\nPublished: %s' \ + "$kind" "$repo" "$tag" "$name" "$published")" + if [[ "$TELEGRAM_INCLUDE_CHANGELOG" == true ]]; then + body="$(jq -r '.body // ""' <<< "$release" | sed -E 's/<[^>]*>/ /g; s/!\[[^]]*\]\([^)]*\)//g')" + allowance=$((4096 - ${#message} - ${#url} - 24)) + if (( allowance > TELEGRAM_CHANGELOG_MAX_LENGTH )); then allowance="$TELEGRAM_CHANGELOG_MAX_LENGTH"; fi + if (( allowance > 0 )) && [[ -n "$body" ]]; then + body="$(printf '%s' "$body" | jq -Rs --argjson max "$allowance" '.[:$max]' -r)" + message+="$(printf '\n\nChangelog:\n%s' "$body")" + fi + fi + message+="$(printf '\n\n%s' "$url")" + # Telegram limit is 4096 Unicode characters, not bytes. + printf '%s' "$message" | jq -Rs '.[:4096]' -r } process_repo() { local repo="$1" releases ids previous new_releases count release id tag message @@ -210,13 +245,17 @@ main() { fi while IFS= read -r url; do [[ -n "$url" ]] || continue - if [[ ! "$url" =~ ^https://github\.com/([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)/?$ ]]; then - log ERROR "Invalid GitHub URL: $url" + if [[ "$url" =~ ^https://(www\.)?github\.com/ ]]; then + url="${url#*github.com/}" + fi + url="${url%/}" + url="${url%.git}" + if [[ ! "$url" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then + log ERROR "Invalid repository: $url" failures=$((failures + 1)) continue fi - repo="${BASH_REMATCH[1]}/${BASH_REMATCH[2]}" - repo="${repo%.git}" + repo="$url" if ! process_repo "$repo"; then failures=$((failures + 1)); fi done < <(jq -r '.repositories[]' "$REPOS_FILE" | sort -u) if (( failures > 0 )); then log ERROR "Failed repositories: $failures"; return 1; fi diff --git a/repos.json b/repos.json index 45e4e48..a9625d2 100644 --- a/repos.json +++ b/repos.json @@ -1,7 +1,6 @@ { "repositories": [ - "https://github.com/LycheeOrg/Lychee", - "https://github.com/firefly-iii/firefly-iii", - "https://github.com/usememos/memos" + "LycheeOrg/Lychee", + "firefly-iii/firefly-iii", + "usememos/memos" ] -}