100 أسطر
3.2 KiB
Bash
100 أسطر
3.2 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# health-check.sh
|
|
# Polls the deployed app's /health and /metrics endpoints every 30 seconds,
|
|
# logs status + response time, and writes a rolling JSON snapshot that
|
|
# dashboard.html polls to render live status.
|
|
#
|
|
# Usage:
|
|
# ./health-check.sh https://myapp.ghaymah.systems
|
|
#
|
|
set -euo pipefail
|
|
|
|
APP_URL="${1:-https://myapp.ghaymah.systems}"
|
|
INTERVAL_SECONDS=30
|
|
LOG_FILE="./health-check.log"
|
|
STATE_FILE="./status.json" # consumed by dashboard.html
|
|
MAX_HISTORY=100 # keep last N checks in status.json
|
|
ALERT_WEBHOOK="${ALERT_WEBHOOK_URL:-}" # optional: Slack/Telegram webhook for alerting
|
|
|
|
FAIL_STREAK=0
|
|
FAIL_THRESHOLD=3 # alert after 3 consecutive failures (90s of downtime)
|
|
|
|
log() {
|
|
echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
send_alert() {
|
|
local message="$1"
|
|
log "ALERT: ${message}"
|
|
if [[ -n "$ALERT_WEBHOOK" ]]; then
|
|
curl -s -X POST -H 'Content-Type: application/json' \
|
|
-d "{\"text\": \"🚨 ${message}\"}" "$ALERT_WEBHOOK" >/dev/null || true
|
|
fi
|
|
}
|
|
|
|
init_state_file() {
|
|
[[ -f "$STATE_FILE" ]] || echo '{"history": []}' > "$STATE_FILE"
|
|
}
|
|
|
|
append_result() {
|
|
local timestamp="$1" status_code="$2" latency_ms="$3" request_count="$4" healthy="$5"
|
|
|
|
local tmp
|
|
tmp=$(mktemp)
|
|
python3 - "$STATE_FILE" "$timestamp" "$status_code" "$latency_ms" "$request_count" "$healthy" "$MAX_HISTORY" <<'PYEOF' > "$tmp"
|
|
import json, sys
|
|
|
|
state_file, ts, code, latency, req_count, healthy, max_hist = sys.argv[1:8]
|
|
try:
|
|
with open(state_file) as f:
|
|
data = json.load(f)
|
|
except (FileNotFoundError, json.JSONDecodeError):
|
|
data = {"history": []}
|
|
|
|
data["history"].append({
|
|
"timestamp": ts,
|
|
"status_code": int(code),
|
|
"latency_ms": float(latency),
|
|
"request_count": int(req_count) if req_count.isdigit() else None,
|
|
"healthy": healthy == "1",
|
|
})
|
|
data["history"] = data["history"][-int(max_hist):]
|
|
data["last_check"] = data["history"][-1]
|
|
print(json.dumps(data, ensure_ascii=False))
|
|
PYEOF
|
|
mv "$tmp" "$STATE_FILE"
|
|
}
|
|
|
|
log "Starting monitor for ${APP_URL} (interval=${INTERVAL_SECONDS}s)"
|
|
init_state_file
|
|
|
|
while true; do
|
|
START_MS=$(date +%s%3N)
|
|
|
|
# -o /dev/null discards body, -w prints status code, --max-time caps hang time
|
|
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "${APP_URL}/health" || echo "000")
|
|
|
|
END_MS=$(date +%s%3N)
|
|
LATENCY_MS=$((END_MS - START_MS))
|
|
|
|
REQUEST_COUNT=$(curl -s --max-time 5 "${APP_URL}/metrics" 2>/dev/null \
|
|
| python3 -c "import sys,json; print(json.load(sys.stdin).get('request_count',''))" 2>/dev/null || echo "")
|
|
|
|
if [[ "$HTTP_CODE" == "200" ]]; then
|
|
log "OK status=${HTTP_CODE} latency=${LATENCY_MS}ms requests=${REQUEST_COUNT}"
|
|
FAIL_STREAK=0
|
|
append_result "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$HTTP_CODE" "$LATENCY_MS" "${REQUEST_COUNT:-0}" 1
|
|
else
|
|
FAIL_STREAK=$((FAIL_STREAK + 1))
|
|
log "FAIL status=${HTTP_CODE} latency=${LATENCY_MS}ms fail_streak=${FAIL_STREAK}"
|
|
append_result "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$HTTP_CODE" "$LATENCY_MS" "${REQUEST_COUNT:-0}" 0
|
|
|
|
if (( FAIL_STREAK >= FAIL_THRESHOLD )); then
|
|
send_alert "${APP_URL} has failed ${FAIL_STREAK} consecutive health checks (last status=${HTTP_CODE})"
|
|
fi
|
|
fi
|
|
|
|
sleep "$INTERVAL_SECONDS"
|
|
done
|