179 lines
4.6 KiB
Go
179 lines
4.6 KiB
Go
package alert
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/mahmoud/db-monitor/collector"
|
|
"github.com/mahmoud/db-monitor/storage"
|
|
)
|
|
|
|
// Engine processes metrics and sends alerts
|
|
type Engine struct {
|
|
store *storage.Store
|
|
cooldown map[string]time.Time
|
|
mu sync.Mutex
|
|
}
|
|
|
|
// NewEngine creates an alert engine
|
|
func NewEngine(store *storage.Store) *Engine {
|
|
return &Engine{
|
|
store: store,
|
|
cooldown: make(map[string]time.Time),
|
|
}
|
|
}
|
|
|
|
// Evaluate checks all rules against the given metrics
|
|
func (e *Engine) Evaluate(m *collector.Metrics) {
|
|
// Always check db_down regardless of rules
|
|
if m.Status == "DOWN" {
|
|
e.fireAlert("db_down", m.DatabaseName,
|
|
fmt.Sprintf("🚨 *PostgreSQL DOWN*\nDatabase: `%s`\nError: %s", m.DatabaseName, m.Error))
|
|
}
|
|
|
|
// Load dynamic rules from storage
|
|
rules, err := e.store.ListAlertRules()
|
|
if err != nil {
|
|
log.Printf("[alert] failed to load rules: %v", err)
|
|
return
|
|
}
|
|
|
|
metricValues := map[string]float64{
|
|
"latency_ms": m.LatencyMs,
|
|
"connection_usage_pct": m.ConnUsagePct,
|
|
"active_connections": float64(m.ActiveConns),
|
|
"total_connections": float64(m.TotalConns),
|
|
"active_queries": float64(m.ActiveQueries),
|
|
"long_running_queries": float64(m.LongRunning),
|
|
"waiting_queries": float64(m.WaitingQueries),
|
|
"cache_hit_ratio": m.CacheHitRatio,
|
|
"active_locks": float64(m.ActiveLocks),
|
|
"waiting_locks": float64(m.WaitingLocks),
|
|
"deadlocks": float64(m.Deadlocks),
|
|
"replication_lag_ms": m.ReplicationLagMs,
|
|
"database_size_bytes": float64(m.DatabaseSizeBytes),
|
|
}
|
|
|
|
metricLabels := map[string]string{
|
|
"latency_ms": "Latency",
|
|
"connection_usage_pct": "Connection Usage",
|
|
"active_connections": "Active Connections",
|
|
"total_connections": "Total Connections",
|
|
"active_queries": "Active Queries",
|
|
"long_running_queries": "Long Running Queries",
|
|
"waiting_queries": "Waiting Queries",
|
|
"cache_hit_ratio": "Cache Hit Ratio",
|
|
"active_locks": "Active Locks",
|
|
"waiting_locks": "Waiting Locks",
|
|
"deadlocks": "Deadlocks",
|
|
"replication_lag_ms": "Replication Lag",
|
|
"database_size_bytes": "Database Size",
|
|
}
|
|
|
|
for _, rule := range rules {
|
|
if !rule.Enabled {
|
|
continue
|
|
}
|
|
|
|
// Filter by database: empty db_name means applies to all
|
|
if rule.DBName != "" && rule.DBName != m.DatabaseName {
|
|
continue
|
|
}
|
|
|
|
val, ok := metricValues[rule.Metric]
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
triggered := false
|
|
switch rule.Operator {
|
|
case ">":
|
|
triggered = val > rule.Threshold
|
|
case ">=":
|
|
triggered = val >= rule.Threshold
|
|
case "<":
|
|
triggered = val < rule.Threshold
|
|
case "<=":
|
|
triggered = val <= rule.Threshold
|
|
case "==":
|
|
triggered = val == rule.Threshold
|
|
}
|
|
|
|
if triggered {
|
|
label := metricLabels[rule.Metric]
|
|
if label == "" {
|
|
label = rule.Metric
|
|
}
|
|
ruleKey := fmt.Sprintf("rule_%d", rule.ID)
|
|
msg := fmt.Sprintf("⚠️ *Alert: %s*\nDatabase: `%s`\n%s: `%.2f` %s `%.2f`",
|
|
label, m.DatabaseName, label, val, rule.Operator, rule.Threshold)
|
|
|
|
e.fireAlertWithCooldown(ruleKey, m.DatabaseName, msg, time.Duration(rule.CooldownMin)*time.Minute)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e *Engine) fireAlert(ruleKey, dbName, message string) {
|
|
e.fireAlertWithCooldown(ruleKey, dbName, message, 5*time.Minute)
|
|
}
|
|
|
|
func (e *Engine) fireAlertWithCooldown(ruleKey, dbName, message string, cooldown time.Duration) {
|
|
key := ruleKey + ":" + dbName
|
|
|
|
e.mu.Lock()
|
|
lastFired, exists := e.cooldown[key]
|
|
if exists && time.Since(lastFired) < cooldown {
|
|
e.mu.Unlock()
|
|
return
|
|
}
|
|
e.cooldown[key] = time.Now()
|
|
e.mu.Unlock()
|
|
|
|
// Load notification channels and send
|
|
go e.sendToChannels(message)
|
|
}
|
|
|
|
func (e *Engine) sendToChannels(message string) {
|
|
// Try Telegram
|
|
ch, err := e.store.GetNotificationChannel("telegram")
|
|
if err != nil || ch == nil || !ch.Enabled {
|
|
return
|
|
}
|
|
|
|
var cfg storage.TelegramConfig
|
|
if err := json.Unmarshal(ch.Config, &cfg); err != nil {
|
|
log.Printf("[alert] invalid telegram config: %v", err)
|
|
return
|
|
}
|
|
|
|
if cfg.BotToken == "" || cfg.ChatID == "" {
|
|
return
|
|
}
|
|
|
|
e.sendTelegram(cfg.BotToken, cfg.ChatID, message)
|
|
}
|
|
|
|
func (e *Engine) sendTelegram(botToken, chatID, message string) {
|
|
apiURL := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", botToken)
|
|
|
|
resp, err := http.PostForm(apiURL, url.Values{
|
|
"chat_id": {chatID},
|
|
"text": {message},
|
|
"parse_mode": {"Markdown"},
|
|
})
|
|
if err != nil {
|
|
log.Printf("[alert] telegram error: %v", err)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
log.Printf("[alert] telegram returned status %d", resp.StatusCode)
|
|
}
|
|
}
|