Improve repository config, changelog notifications and API resilience

.env.example

@@ -8,3 +8,7 @@ GITHUB_MAX_PAGES=3
 REPOS_FILE=repos.json
 STATE_FILE=state.json
 INCLUDE_PRERELEASES=false
+
+TELEGRAM_INCLUDE_CHANGELOG=true
+TELEGRAM_CHANGELOG_MAX_LENGTH=1800
+STATE_MAX_IDS=500

README.md

@@ -65,3 +65,12 @@ through command-line arguments. The state file is read directly by `jq`.
 This prevents `Argument list too long` for repositories with large release notes.
 
 Run `bash -n monitor.sh` and `./monitor.sh --dry-run --verbose` to verify.
+
+### Release notes and retention
+
+`repos.json` uses `owner/repo` names (legacy full GitHub URLs remain accepted).
+`TELEGRAM_INCLUDE_CHANGELOG=true` includes a shortened release body in Telegram messages.
+`TELEGRAM_CHANGELOG_MAX_LENGTH=1800` limits the changelog; Telegram messages are capped at 4096 characters.
+`STATE_MAX_IDS=500` bounds stored release IDs per repository. Keep this larger than the number of releases that may appear in the fetched pages; old releases returning after pruning may otherwise be notified again.
+
+GitHub HTTP 403/429 responses log rate-limit headers and fail without changing the repository state. Mid-pagination errors also fail the repository without partially processing it. Tags without GitHub releases are not tracked.

monitor.sh

@@ -57,6 +57,9 @@ set +a
 : "${REPOS_FILE:=repos.json}"
 : "${STATE_FILE:=state.json}"
 : "${INCLUDE_PRERELEASES:=false}"
+: "${TELEGRAM_INCLUDE_CHANGELOG:=true}"
+: "${TELEGRAM_CHANGELOG_MAX_LENGTH:=1800}"
+: "${STATE_MAX_IDS:=500}"
 if [[ "$CLI_PRERELEASES" == true ]]; then INCLUDE_PRERELEASES=true; fi
 resolve_path() { if [[ "$1" == /* ]]; then printf '%s\n' "$1"; else printf '%s/%s\n' "$SCRIPT_DIR" "$1"; fi; }
 REPOS_FILE="$(resolve_path "$REPOS_FILE")"
@@ -67,12 +70,18 @@ for cmd in curl jq flock mktemp; do command -v "$cmd" >/dev/null 2>&1 || die "Mi
 (( GITHUB_PER_PAGE <= 100 )) || die "GITHUB_PER_PAGE must be <= 100"
 [[ "$GITHUB_MAX_PAGES" =~ ^[1-9][0-9]*$ ]] || die "Invalid GITHUB_MAX_PAGES"
 [[ "$INCLUDE_PRERELEASES" == true || "$INCLUDE_PRERELEASES" == false ]] || die "Invalid INCLUDE_PRERELEASES"
+for var in TELEGRAM_CHANGELOG_MAX_LENGTH STATE_MAX_IDS; do
+    [[ "${!var}" =~ ^[1-9][0-9]*$ ]] || die "Invalid $var"
+done
+[[ "$TELEGRAM_INCLUDE_CHANGELOG" == true || "$TELEGRAM_INCLUDE_CHANGELOG" == false ]] || die 'Invalid TELEGRAM_INCLUDE_CHANGELOG'
 if [[ "$TEST_TELEGRAM" == true || ( "$INIT" == false && "$DRY_RUN" == false ) ]]; then
     [[ -n "$TELEGRAM_BOT_TOKEN" && -n "$TELEGRAM_CHAT_ID" ]] || die "Telegram credentials are missing"
 fi
 umask 077
 CURL_OPTIONS=(--silent --show-error --fail --location --connect-timeout 10 --max-time "$REQUEST_TIMEOUT")
 if [[ -n "$SOCKS5_PROXY" ]]; then CURL_OPTIONS+=(--proxy "$SOCKS5_PROXY"); fi
+GITHUB_CURL_OPTIONS=(--silent --show-error --location --connect-timeout 10 --max-time "$REQUEST_TIMEOUT")
+if [[ -n "$SOCKS5_PROXY" ]]; then GITHUB_CURL_OPTIONS+=(--proxy "$SOCKS5_PROXY"); fi
 GITHUB_HEADERS=(-H 'Accept: application/vnd.github+json' -H 'X-GitHub-Api-Version: 2022-11-28' -H 'User-Agent: github-release-monitor')
 if [[ -n "$GITHUB_TOKEN" ]]; then GITHUB_HEADERS+=(-H "Authorization: Bearer ${GITHUB_TOKEN}"); fi
 debug() { if [[ "$VERBOSE" == true ]]; then log DEBUG "$1"; fi; }
@@ -89,33 +98,47 @@ send_telegram() {
 test_telegram() {
     local message
     message="$(printf 'GitHub Release Monitor: Test notification\nHost: %s\nTime: %s\nStatus: Telegram connection successful' "$(hostname)" "$(date '+%Y-%m-%d %H:%M:%S %Z')")"
+    if [[ "$TELEGRAM_INCLUDE_CHANGELOG" == true ]]; then
+        message+=$'\n\nChangelog:\nTest changelog rendering enabled.'
+    fi
     log INFO 'Sending test Telegram notification'
     if send_telegram "$message"; then log INFO 'Test notification sent'; else log ERROR 'Test notification failed'; return 1; fi
 }
 fetch_releases() {
-    local repo="$1" page count response_file releases_file
+    local repo="$1" page count response_file releases_file headers_file http_code retry_after remaining reset
     response_file="$(mktemp)" || return 1
     releases_file="$(mktemp)" || { rm -f "$response_file"; return 1; }
+    headers_file="$(mktemp)" || { rm -f "$response_file" "$releases_file"; return 1; }
 
     for ((page=1; page<=GITHUB_MAX_PAGES; page++)); do
         debug "Fetching $repo page=$page"
-        if ! curl "${CURL_OPTIONS[@]}" "${GITHUB_HEADERS[@]}" \
+        if ! http_code="$(curl "${GITHUB_CURL_OPTIONS[@]}" "${GITHUB_HEADERS[@]}" \
+            --dump-header "$headers_file" --write-out '%{http_code}' \
             --output "$response_file" \
-            "${GITHUB_API}/repos/${repo}/releases?per_page=${GITHUB_PER_PAGE}&page=${page}"; then
+            "${GITHUB_API}/repos/${repo}/releases?per_page=${GITHUB_PER_PAGE}&page=${page}")"; then
-            rm -f "$response_file" "$releases_file"
+            rm -f "$response_file" "$releases_file" "$headers_file"
+            return 1
+        fi
+        if [[ "$http_code" != 200 ]]; then
+            retry_after="$(awk 'BEGIN{IGNORECASE=1} tolower($1)=="retry-after:" {gsub("\r", "", $2); print $2}' "$headers_file" | tail -n 1)"
+            remaining="$(awk 'tolower($1)=="x-ratelimit-remaining:" {gsub("\r", "", $2); print $2}' "$headers_file" | tail -n 1)"
+            reset="$(awk 'tolower($1)=="x-ratelimit-reset:" {gsub("\r", "", $2); print $2}' "$headers_file" | tail -n 1)"
+            log ERROR "GitHub HTTP $http_code for $repo; rate_remaining=${remaining:-unknown}; rate_reset=${reset:-unknown}; retry_after=${retry_after:-unknown}"
+            rm -f "$response_file" "$releases_file" "$headers_file"
             return 1
         fi
         if ! count="$(jq -er 'if type == "array" then length else error("Invalid releases response") end' "$response_file")"; then
-            rm -f "$response_file" "$releases_file"
+            rm -f "$response_file" "$releases_file" "$headers_file"
             return 1
         fi
         if ! jq -c --argjson prereleases "$INCLUDE_PRERELEASES" '
             .[] | select(.draft == false) |
             select($prereleases or (.prerelease == false)) |
             {id: (.id | tostring), tag: .tag_name, name: .name,
-             url: .html_url, published: .published_at, prerelease: .prerelease}
+             url: .html_url, published: .published_at, prerelease: .prerelease,
+             body: (.body // "")}
         ' "$response_file" >> "$releases_file"; then
-            rm -f "$response_file" "$releases_file"
+            rm -f "$response_file" "$releases_file" "$headers_file"
             return 1
         fi
         if (( count < GITHUB_PER_PAGE )); then break; fi
@@ -123,22 +146,22 @@ fetch_releases() {
 
     local status=0
     jq -sc '.' "$releases_file" || status=$?
-    rm -f "$response_file" "$releases_file"
+    rm -f "$response_file" "$releases_file" "$headers_file"
     return "$status"
 }
 
 write_state() {
     local repo="$1" ids="$2" mode="$3" temp
     temp="$(mktemp "${STATE_FILE}.tmp.XXXXXX")" || return 1
-    if ! jq --arg repo "$repo" --arg mode "$mode" --slurpfile ids /dev/stdin '
+    if ! jq --arg repo "$repo" --arg mode "$mode" --argjson limit "$STATE_MAX_IDS" --slurpfile ids /dev/stdin '
         ($ids[0] // []) as $new_ids |
-        if $mode == "replace" then .[$repo] = $new_ids
+        if $mode == "replace" then .[$repo] = ($new_ids | unique | .[-$limit:])
-        else .[$repo] = ((.[$repo] // []) + $new_ids | unique) end
+        else .[$repo] = ((.[$repo] // []) + $new_ids | unique | .[-$limit:]) end
     ' "$STATE_FILE" <<< "$ids" > "$temp"; then rm -f "$temp"; return 1; fi
     if ! mv -f "$temp" "$STATE_FILE"; then rm -f "$temp"; return 1; fi
 }
 format_message() {
-    local repo="$1" release="$2" tag name url published kind
+    local repo="$1" release="$2" tag name url published kind body message allowance
     tag="$(jq -r '.tag // "unknown"' <<< "$release")"
     name="$(jq -r '.name // empty' <<< "$release")"
     url="$(jq -r '.url // empty' <<< "$release")"
@@ -146,8 +169,20 @@ format_message() {
     [[ -n "$name" ]] || name="$tag"
     kind=Release
     if [[ "$(jq -r '.prerelease' <<< "$release")" == true ]]; then kind=Prerelease; fi
-    printf 'GitHub: New %s\nRepository: %s\nVersion: %s\nName: %s\nPublished: %s\nURL: %s' \
+    message="$(printf 'GitHub: New %s\nRepository: %s\nVersion: %s\nName: %s\nPublished: %s' \
-        "$kind" "$repo" "$tag" "$name" "$published" "$url"
+        "$kind" "$repo" "$tag" "$name" "$published")"
+    if [[ "$TELEGRAM_INCLUDE_CHANGELOG" == true ]]; then
+        body="$(jq -r '.body // ""' <<< "$release" | sed -E 's/<[^>]*>/ /g; s/!\[[^]]*\]\([^)]*\)//g')"
+        allowance=$((4096 - ${#message} - ${#url} - 24))
+        if (( allowance > TELEGRAM_CHANGELOG_MAX_LENGTH )); then allowance="$TELEGRAM_CHANGELOG_MAX_LENGTH"; fi
+        if (( allowance > 0 )) && [[ -n "$body" ]]; then
+            body="$(printf '%s' "$body" | jq -Rs --argjson max "$allowance" '.[:$max]' -r)"
+            message+="$(printf '\n\nChangelog:\n%s' "$body")"
+        fi
+    fi
+    message+="$(printf '\n\n%s' "$url")"
+    # Telegram limit is 4096 Unicode characters, not bytes.
+    printf '%s' "$message" | jq -Rs '.[:4096]' -r
 }
 process_repo() {
     local repo="$1" releases ids previous new_releases count release id tag message
@@ -210,13 +245,17 @@ main() {
     fi
     while IFS= read -r url; do
         [[ -n "$url" ]] || continue
-        if [[ ! "$url" =~ ^https://github\.com/([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)/?$ ]]; then
+        if [[ "$url" =~ ^https://(www\.)?github\.com/ ]]; then
-            log ERROR "Invalid GitHub URL: $url"
+            url="${url#*github.com/}"
+        fi
+        url="${url%/}"
+        url="${url%.git}"
+        if [[ ! "$url" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then
+            log ERROR "Invalid repository: $url"
             failures=$((failures + 1))
             continue
         fi
-        repo="${BASH_REMATCH[1]}/${BASH_REMATCH[2]}"
+        repo="$url"
-        repo="${repo%.git}"
         if ! process_repo "$repo"; then failures=$((failures + 1)); fi
     done < <(jq -r '.repositories[]' "$REPOS_FILE" | sort -u)
     if (( failures > 0 )); then log ERROR "Failed repositories: $failures"; return 1; fi

repos.json

@@ -1,7 +1,6 @@
 {
   "repositories": [
-    "https://github.com/LycheeOrg/Lychee",
+    "LycheeOrg/Lychee",
-    "https://github.com/firefly-iii/firefly-iii",
+    "firefly-iii/firefly-iii",
-    "https://github.com/usememos/memos"
+    "usememos/memos"
   ]
-}