Files
2026-10-30 16:35:46 +02:00

179 lines
4.6 KiB
Go

package alert
import (
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"sync"
"time"
"github.com/mahmoud/db-monitor/collector"
"github.com/mahmoud/db-monitor/storage"
)
// Engine processes metrics and sends alerts
type Engine struct {
store *storage.Store
cooldown map[string]time.Time
mu sync.Mutex
}
// NewEngine creates an alert engine
func NewEngine(store *storage.Store) *Engine {
return &Engine{
store: store,
cooldown: make(map[string]time.Time),
}
}
// Evaluate checks all rules against the given metrics
func (e *Engine) Evaluate(m *collector.Metrics) {
// Always check db_down regardless of rules
if m.Status == "DOWN" {
e.fireAlert("db_down", m.DatabaseName,
fmt.Sprintf("🚨 *PostgreSQL DOWN*\nDatabase: `%s`\nError: %s", m.DatabaseName, m.Error))
}
// Load dynamic rules from storage
rules, err := e.store.ListAlertRules()
if err != nil {
log.Printf("[alert] failed to load rules: %v", err)
return
}
metricValues := map[string]float64{
"latency_ms": m.LatencyMs,
"connection_usage_pct": m.ConnUsagePct,
"active_connections": float64(m.ActiveConns),
"total_connections": float64(m.TotalConns),
"active_queries": float64(m.ActiveQueries),
"long_running_queries": float64(m.LongRunning),
"waiting_queries": float64(m.WaitingQueries),
"cache_hit_ratio": m.CacheHitRatio,
"active_locks": float64(m.ActiveLocks),
"waiting_locks": float64(m.WaitingLocks),
"deadlocks": float64(m.Deadlocks),
"replication_lag_ms": m.ReplicationLagMs,
"database_size_bytes": float64(m.DatabaseSizeBytes),
}
metricLabels := map[string]string{
"latency_ms": "Latency",
"connection_usage_pct": "Connection Usage",
"active_connections": "Active Connections",
"total_connections": "Total Connections",
"active_queries": "Active Queries",
"long_running_queries": "Long Running Queries",
"waiting_queries": "Waiting Queries",
"cache_hit_ratio": "Cache Hit Ratio",
"active_locks": "Active Locks",
"waiting_locks": "Waiting Locks",
"deadlocks": "Deadlocks",
"replication_lag_ms": "Replication Lag",
"database_size_bytes": "Database Size",
}
for _, rule := range rules {
if !rule.Enabled {
continue
}
// Filter by database: empty db_name means applies to all
if rule.DBName != "" && rule.DBName != m.DatabaseName {
continue
}
val, ok := metricValues[rule.Metric]
if !ok {
continue
}
triggered := false
switch rule.Operator {
case ">":
triggered = val > rule.Threshold
case ">=":
triggered = val >= rule.Threshold
case "<":
triggered = val < rule.Threshold
case "<=":
triggered = val <= rule.Threshold
case "==":
triggered = val == rule.Threshold
}
if triggered {
label := metricLabels[rule.Metric]
if label == "" {
label = rule.Metric
}
ruleKey := fmt.Sprintf("rule_%d", rule.ID)
msg := fmt.Sprintf("⚠️ *Alert: %s*\nDatabase: `%s`\n%s: `%.2f` %s `%.2f`",
label, m.DatabaseName, label, val, rule.Operator, rule.Threshold)
e.fireAlertWithCooldown(ruleKey, m.DatabaseName, msg, time.Duration(rule.CooldownMin)*time.Minute)
}
}
}
func (e *Engine) fireAlert(ruleKey, dbName, message string) {
e.fireAlertWithCooldown(ruleKey, dbName, message, 5*time.Minute)
}
func (e *Engine) fireAlertWithCooldown(ruleKey, dbName, message string, cooldown time.Duration) {
key := ruleKey + ":" + dbName
e.mu.Lock()
lastFired, exists := e.cooldown[key]
if exists && time.Since(lastFired) < cooldown {
e.mu.Unlock()
return
}
e.cooldown[key] = time.Now()
e.mu.Unlock()
// Load notification channels and send
go e.sendToChannels(message)
}
func (e *Engine) sendToChannels(message string) {
// Try Telegram
ch, err := e.store.GetNotificationChannel("telegram")
if err != nil || ch == nil || !ch.Enabled {
return
}
var cfg storage.TelegramConfig
if err := json.Unmarshal(ch.Config, &cfg); err != nil {
log.Printf("[alert] invalid telegram config: %v", err)
return
}
if cfg.BotToken == "" || cfg.ChatID == "" {
return
}
e.sendTelegram(cfg.BotToken, cfg.ChatID, message)
}
func (e *Engine) sendTelegram(botToken, chatID, message string) {
apiURL := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", botToken)
resp, err := http.PostForm(apiURL, url.Values{
"chat_id": {chatID},
"text": {message},
"parse_mode": {"Markdown"},
})
if err != nil {
log.Printf("[alert] telegram error: %v", err)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Printf("[alert] telegram returned status %d", resp.StatusCode)
}
}