#!/usr/bin/env bash # # health-check.sh # Polls the deployed app's /health and /metrics endpoints every 30 seconds, # logs status + response time, and writes a rolling JSON snapshot that # dashboard.html polls to render live status. # # Usage: # ./health-check.sh https://myapp.ghaymah.systems # set -euo pipefail APP_URL="${1:-https://myapp.ghaymah.systems}" INTERVAL_SECONDS=30 LOG_FILE="./health-check.log" STATE_FILE="./status.json" # consumed by dashboard.html MAX_HISTORY=100 # keep last N checks in status.json ALERT_WEBHOOK="${ALERT_WEBHOOK_URL:-}" # optional: Slack/Telegram webhook for alerting FAIL_STREAK=0 FAIL_THRESHOLD=3 # alert after 3 consecutive failures (90s of downtime) log() { echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" | tee -a "$LOG_FILE" } send_alert() { local message="$1" log "ALERT: ${message}" if [[ -n "$ALERT_WEBHOOK" ]]; then curl -s -X POST -H 'Content-Type: application/json' \ -d "{\"text\": \"🚨 ${message}\"}" "$ALERT_WEBHOOK" >/dev/null || true fi } init_state_file() { [[ -f "$STATE_FILE" ]] || echo '{"history": []}' > "$STATE_FILE" } append_result() { local timestamp="$1" status_code="$2" latency_ms="$3" request_count="$4" healthy="$5" local tmp tmp=$(mktemp) python3 - "$STATE_FILE" "$timestamp" "$status_code" "$latency_ms" "$request_count" "$healthy" "$MAX_HISTORY" <<'PYEOF' > "$tmp" import json, sys state_file, ts, code, latency, req_count, healthy, max_hist = sys.argv[1:8] try: with open(state_file) as f: data = json.load(f) except (FileNotFoundError, json.JSONDecodeError): data = {"history": []} data["history"].append({ "timestamp": ts, "status_code": int(code), "latency_ms": float(latency), "request_count": int(req_count) if req_count.isdigit() else None, "healthy": healthy == "1", }) data["history"] = data["history"][-int(max_hist):] data["last_check"] = data["history"][-1] print(json.dumps(data, ensure_ascii=False)) PYEOF mv "$tmp" "$STATE_FILE" } log "Starting monitor for ${APP_URL} (interval=${INTERVAL_SECONDS}s)" init_state_file while true; do START_MS=$(date +%s%3N) # -o /dev/null discards body, -w prints status code, --max-time caps hang time HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "${APP_URL}/health" || echo "000") END_MS=$(date +%s%3N) LATENCY_MS=$((END_MS - START_MS)) REQUEST_COUNT=$(curl -s --max-time 5 "${APP_URL}/metrics" 2>/dev/null \ | python3 -c "import sys,json; print(json.load(sys.stdin).get('request_count',''))" 2>/dev/null || echo "") if [[ "$HTTP_CODE" == "200" ]]; then log "OK status=${HTTP_CODE} latency=${LATENCY_MS}ms requests=${REQUEST_COUNT}" FAIL_STREAK=0 append_result "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$HTTP_CODE" "$LATENCY_MS" "${REQUEST_COUNT:-0}" 1 else FAIL_STREAK=$((FAIL_STREAK + 1)) log "FAIL status=${HTTP_CODE} latency=${LATENCY_MS}ms fail_streak=${FAIL_STREAK}" append_result "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$HTTP_CODE" "$LATENCY_MS" "${REQUEST_COUNT:-0}" 0 if (( FAIL_STREAK >= FAIL_THRESHOLD )); then send_alert "${APP_URL} has failed ${FAIL_STREAK} consecutive health checks (last status=${HTTP_CODE})" fi fi sleep "$INTERVAL_SECONDS" done