Upload files to "q1-deploy-monitor"

q1-deploy-monitor/health-check.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+#
+# health-check.sh
+# Polls the deployed app's /health and /metrics endpoints every 30 seconds,
+# logs status + response time, and writes a rolling JSON snapshot that
+# dashboard.html polls to render live status.
+#
+# Usage:
+#   ./health-check.sh https://myapp.ghaymah.systems
+#
+set -euo pipefail
+
+APP_URL="${1:-https://myapp.ghaymah.systems}"
+INTERVAL_SECONDS=30
+LOG_FILE="./health-check.log"
+STATE_FILE="./status.json"          # consumed by dashboard.html
+MAX_HISTORY=100                     # keep last N checks in status.json
+ALERT_WEBHOOK="${ALERT_WEBHOOK_URL:-}"   # optional: Slack/Telegram webhook for alerting
+
+FAIL_STREAK=0
+FAIL_THRESHOLD=3   # alert after 3 consecutive failures (90s of downtime)
+
+log() {
+  echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*" | tee -a "$LOG_FILE"
+}
+
+send_alert() {
+  local message="$1"
+  log "ALERT: ${message}"
+  if [[ -n "$ALERT_WEBHOOK" ]]; then
+    curl -s -X POST -H 'Content-Type: application/json' \
+      -d "{\"text\": \"🚨 ${message}\"}" "$ALERT_WEBHOOK" >/dev/null || true
+  fi
+}
+
+init_state_file() {
+  [[ -f "$STATE_FILE" ]] || echo '{"history": []}' > "$STATE_FILE"
+}
+
+append_result() {
+  local timestamp="$1" status_code="$2" latency_ms="$3" request_count="$4" healthy="$5"
+
+  local tmp
+  tmp=$(mktemp)
+  python3 - "$STATE_FILE" "$timestamp" "$status_code" "$latency_ms" "$request_count" "$healthy" "$MAX_HISTORY" <<'PYEOF' > "$tmp"
+import json, sys
+
+state_file, ts, code, latency, req_count, healthy, max_hist = sys.argv[1:8]
+try:
+    with open(state_file) as f:
+        data = json.load(f)
+except (FileNotFoundError, json.JSONDecodeError):
+    data = {"history": []}
+
+data["history"].append({
+    "timestamp": ts,
+    "status_code": int(code),
+    "latency_ms": float(latency),
+    "request_count": int(req_count) if req_count.isdigit() else None,
+    "healthy": healthy == "1",
+})
+data["history"] = data["history"][-int(max_hist):]
+data["last_check"] = data["history"][-1]
+print(json.dumps(data, ensure_ascii=False))
+PYEOF
+  mv "$tmp" "$STATE_FILE"
+}
+
+log "Starting monitor for ${APP_URL} (interval=${INTERVAL_SECONDS}s)"
+init_state_file
+
+while true; do
+  START_MS=$(date +%s%3N)
+
+  # -o /dev/null discards body, -w prints status code, --max-time caps hang time
+  HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "${APP_URL}/health" || echo "000")
+
+  END_MS=$(date +%s%3N)
+  LATENCY_MS=$((END_MS - START_MS))
+
+  REQUEST_COUNT=$(curl -s --max-time 5 "${APP_URL}/metrics" 2>/dev/null \
+    | python3 -c "import sys,json; print(json.load(sys.stdin).get('request_count',''))" 2>/dev/null || echo "")
+
+  if [[ "$HTTP_CODE" == "200" ]]; then
+    log "OK   status=${HTTP_CODE} latency=${LATENCY_MS}ms requests=${REQUEST_COUNT}"
+    FAIL_STREAK=0
+    append_result "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$HTTP_CODE" "$LATENCY_MS" "${REQUEST_COUNT:-0}" 1
+  else
+    FAIL_STREAK=$((FAIL_STREAK + 1))
+    log "FAIL status=${HTTP_CODE} latency=${LATENCY_MS}ms fail_streak=${FAIL_STREAK}"
+    append_result "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$HTTP_CODE" "$LATENCY_MS" "${REQUEST_COUNT:-0}" 0
+
+    if (( FAIL_STREAK >= FAIL_THRESHOLD )); then
+      send_alert "${APP_URL} has failed ${FAIL_STREAK} consecutive health checks (last status=${HTTP_CODE})"
+    fi
+  fi
+
+  sleep "$INTERVAL_SECONDS"
+done