commit 4fd4bb72e10d210a5921ea8099bb1d35dab2d878 Author: mac Date: Mon Jul 27 13:44:44 2026 +0300 feat: complete initial project structure for API and Monitoring Dashboards diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..da09dc7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,60 @@ +name: Build and Deploy to ghaymah.systems + +on: + push: + branches: + - main + +env: + REGISTRY: registry.ghaymah.systems + IMAGE_NAME: ${{ github.repository }}/myapp-api + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to ghaymah Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.GHAYMAH_USERNAME }} + password: ${{ secrets.GHAYMAH_TOKEN }} + + - name: Build and push Docker image (Staging Tag) + uses: docker/build-push-action@v5 + with: + context: ./api + push: true + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:staging-${{ github.sha }} + + deploy-staging: + needs: build-and-push + runs-on: ubuntu-latest + environment: staging + steps: + - name: Deploy to Staging + run: | + echo "Deploying staging-${{ github.sha }} to ghaymah systems..." + # Example CLI command: ghaymah deploy --image ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:staging-${{ github.sha }} --env staging + + deploy-production: + needs: deploy-staging + runs-on: ubuntu-latest + environment: + name: production + # Manual approval is configured at the GitHub Environment level in repo settings. + steps: + - name: Retag image for production + uses: docker/build-push-action@v5 + with: + context: ./api + push: true + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:production-${{ github.sha }},${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + + - name: Deploy to Production + run: | + echo "Deploying production-${{ github.sha }} to ghaymah systems..." + # Example CLI command: ghaymah deploy --image ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:production-${{ github.sha }} --env production diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..371c12a --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +venv/ +__pycache__/ +*.pyc +.env +monitor.log +api.log +monitor_sh.log +.DS_Store diff --git a/api/Dockerfile b/api/Dockerfile new file mode 100644 index 0000000..35c6654 --- /dev/null +++ b/api/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt requirements.txt +RUN pip install --no-cache-dir -r requirements.txt +COPY app.py app.py +EXPOSE 8080 +CMD ["python", "app.py"] diff --git a/api/app.py b/api/app.py new file mode 100644 index 0000000..7d4848f --- /dev/null +++ b/api/app.py @@ -0,0 +1,10 @@ +from flask import Flask, jsonify + +app = Flask(__name__) + +@app.route('/health') +def health(): + return jsonify({"status": "ok"}) + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=8080) diff --git a/api/requirements.txt b/api/requirements.txt new file mode 100644 index 0000000..95fef4e --- /dev/null +++ b/api/requirements.txt @@ -0,0 +1 @@ +Flask==3.0.3 diff --git a/architecture/block_storage.md b/architecture/block_storage.md new file mode 100644 index 0000000..1dd5abf --- /dev/null +++ b/architecture/block_storage.md @@ -0,0 +1,22 @@ +# Using ghaymah Block Storage for Stateful Workloads + +Containers are inherently stateless; they lose their local filesystem data when they are destroyed or rescheduled. For applications that require persistent data (Stateful Workloads), we utilize **ghaymah Block Storage**. + +## 1. What is Block Storage? +Block Storage provides persistent, highly available disk volumes that can be attached to containers. Unlike object storage (S3), it behaves like a physical hard drive mounted to the OS. + +## 2. Use Cases in our Architecture +While our API application is mostly stateless, certain workloads require persistence: +- **Local Caching:** If a container downloads large datasets or machine learning models upon startup, these can be stored on Block Storage so subsequent container restarts are faster. +- **Session Data / Logs:** If we are writing complex audit logs that haven't yet been shipped to a centralized logging service. +- **Databases:** If running a self-managed database (e.g., PostgreSQL or Redis) within a container, Block Storage is mandatory to prevent data loss. + +## 3. Configuration & Mounting +When deploying via the `ghaymah deploy` CLI or dashboard, we specify a volume mount: +```yaml +volumes: + - name: my-persistent-data + size: 50GB + mountPath: /mnt/data +``` +Inside the container, the application can simply read/write files to `/mnt/data/` knowing the data will survive container restarts. diff --git a/architecture/capacity.md b/architecture/capacity.md new file mode 100644 index 0000000..6e4c26d --- /dev/null +++ b/architecture/capacity.md @@ -0,0 +1,24 @@ +# Capacity Planning: Handling 15,000 req/s + +To ensure high availability and responsiveness under a load of 15,000 requests per second, we must calculate the required number of container instances. + +## Base Assumptions +- **Target Load:** 15,000 req/s +- **Max Capacity per Container:** 500 req/s +- **Safety Margin:** 30% + +## Calculation +1. **Effective Capacity per Container:** + To maintain a 30% margin, we calculate the effective capacity each container should handle before we consider scaling out. + `500 req/s * (1 - 0.30) = 350 req/s` + +2. **Total Containers Required:** + Divide the total expected load by the effective capacity per container. + `15,000 req/s / 350 req/s per container ≈ 42.85` + +3. **Rounding Up:** + We cannot have a fraction of a container, so we always round up to the next whole number. + `ceil(42.85) = 43 containers` + +## Conclusion +To safely handle 15,000 req/s while maintaining a 30% safety margin (which helps absorb sudden traffic spikes or the failure of a few containers), the auto-scaling group should be configured to maintain a baseline of **43 containers** during peak load. diff --git a/architecture/cold_start.md b/architecture/cold_start.md new file mode 100644 index 0000000..e7fdb69 --- /dev/null +++ b/architecture/cold_start.md @@ -0,0 +1,20 @@ +# Cold Start Strategy + +When auto-scaling responds to a traffic spike, new containers must be initialized. The time it takes from the scaling decision to the container actually serving requests is the "cold start" latency. + +To minimize this delay and prevent dropped requests, we implement the following strategy: + +## 1. Lightweight Base Images +- Use Alpine or distroless base images (e.g., `python:3.11-alpine`). +- Smaller images pull faster from the Container Registry over the network. + +## 2. Pre-warming (Buffer Pool) +- Configure the Auto-Scaling Group to always maintain a "buffer" of idle containers (e.g., 10% of the current required capacity). +- If we need 43 containers for active load, we run ~47 containers. When traffic spikes, these 4 idle containers can serve requests instantly while the ASG provisions new ones. + +## 3. Lazy Loading & Readiness Probes +- Defer non-critical initialization (like building large in-memory caches) until *after* the container has started accepting requests. +- Configure Kubernetes/ghaymah readiness probes to accurately reflect when the app is ready to serve traffic, ensuring the load balancer doesn't route traffic to a container that is still booting. + +## 4. Keep-Alive & Connection Pooling +- Ensure idle containers aren't prematurely terminated. Keep database connections alive in a connection pool to avoid the latency of establishing new TCP handshakes during a sudden burst. diff --git a/architecture/diagram.mmd b/architecture/diagram.mmd new file mode 100644 index 0000000..37b1457 --- /dev/null +++ b/architecture/diagram.mmd @@ -0,0 +1,26 @@ +```mermaid +graph TD + Client((Client Requests
15,000 req/s)) --> WAF[Web Application Firewall] + WAF --> LB[ghaymah Load Balancer] + + subgraph Auto-Scaling Group + direction LR + LB -->|Traffic Distribution| API1[myapp-api Container 1
~350 req/s] + LB --> API2[myapp-api Container 2
~350 req/s] + LB --> API3[myapp-api Container 3] + LB -.-> APIN[myapp-api Container N
Total: 43 Containers] + end + + API1 --> BS1[(ghaymah Block Storage
/mnt/data)] + API2 --> BS2[(ghaymah Block Storage
/mnt/data)] + API3 --> BS3[(ghaymah Block Storage
/mnt/data)] + APIN --> BSN[(ghaymah Block Storage
/mnt/data)] + + classDef container fill:#e3f2fd,stroke:#1565c0,stroke-width:2px; + classDef lb fill:#fff3e0,stroke:#e65100,stroke-width:2px; + classDef storage fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px; + + class API1,API2,API3,APIN container; + class LB lb; + class BS1,BS2,BS3,BSN storage; +``` diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000..dab747a --- /dev/null +++ b/ci/README.md @@ -0,0 +1,17 @@ +# Environments: Staging vs Production + +In our deployment pipeline, we utilize two main environments: **Staging** and **Production**. Understanding the differences between them is crucial for safe software delivery. + +## 1. Staging Environment (`staging`) +- **Purpose:** A pre-production area for QA testing, integration testing, and final review by stakeholders before a release goes live. +- **Data:** Uses dummy, sanitized, or replicated data. NEVER connects to the live production database. +- **Access:** Restricted to internal team members, developers, and QA testers. Often protected by VPN, IP whitelisting, or Basic Auth. +- **Scale:** Typically scaled down (fewer containers, smaller database instances) to save costs, as it doesn't need to handle user traffic. +- **Deployment:** Automatic upon merging code to the `main` branch. + +## 2. Production Environment (`production`) +- **Purpose:** The live environment that real users interact with. +- **Data:** Contains live, sensitive, real user data. Strict access controls and backups are enforced. +- **Access:** Publicly accessible (for web apps/APIs). Infrastructure access is strictly limited to authorized SREs/DevOps personnel. +- **Scale:** Scaled up to handle expected user load, with Auto-Scaling policies enabled to handle traffic spikes. +- **Deployment:** Requires a **Manual Approval** step in the CI/CD pipeline (e.g., in GitHub Actions) to ensure that the code deployed to staging has been properly vetted and approved for live release. diff --git a/ci/ghaymah_cli.md b/ci/ghaymah_cli.md new file mode 100644 index 0000000..51042d5 --- /dev/null +++ b/ci/ghaymah_cli.md @@ -0,0 +1,44 @@ +# Integrating with the ghaymah CLI + +To manage and deploy applications to ghaymah.systems from your local machine or CI/CD pipeline, you need to use the `ghaymah` CLI. + +## 1. Installation +Depending on your OS, install the CLI (example for macOS/Linux): +```bash +curl -sL https://cli.ghaymah.systems/install.sh | bash +``` + +## 2. Authentication +Log in to your ghaymah account: +```bash +ghaymah login +``` +This will open a browser window to authenticate. If you are in a CI/CD environment (headless), use a token: +```bash +ghaymah login --token $GHAYMAH_TOKEN +``` + +## 3. Pushing Images to ghaymah Container Registry +Authenticate Docker with the ghaymah registry: +```bash +docker login registry.ghaymah.systems -u $GHAYMAH_USERNAME -p $GHAYMAH_TOKEN +``` +Build and push your image: +```bash +docker build -t registry.ghaymah.systems/my-org/myapp-api:v1 . +docker push registry.ghaymah.systems/my-org/myapp-api:v1 +``` + +## 4. Deploying the Application +Once the image is in the registry, deploy it using the CLI: +```bash +ghaymah deploy \ + --name myapp-api \ + --image registry.ghaymah.systems/my-org/myapp-api:v1 \ + --port 8080 \ + --env production +``` +You can also monitor logs in real-time: +```bash +ghaymah logs myapp-api --follow +``` diff --git a/dashboard/index.html b/dashboard/index.html new file mode 100644 index 0000000..90d02b5 --- /dev/null +++ b/dashboard/index.html @@ -0,0 +1,35 @@ + + + + + Ghaymah API Monitoring + + + +

نظام المراقبة السحابي (API)

+ +
+
+
حالة الخدمة
+
+ + جاري الفحص... +
+
+ +
+
زمن الاستجابة
+
+ - ms +
+
+ +
+
إجمالي الطلبات
+
0
+
+
+ + + + diff --git a/dashboard/script.js b/dashboard/script.js new file mode 100644 index 0000000..019acb3 --- /dev/null +++ b/dashboard/script.js @@ -0,0 +1,35 @@ +let requestCount = 0; + +async function checkHealth() { + const startTime = Date.now(); + try { + const response = await fetch('http://localhost:8080/health'); + const latency = Date.now() - startTime; + requestCount++; + + if (response.ok) { + document.getElementById('status').innerHTML = ` + + متصل ويعمل + `; + } else { + document.getElementById('status').innerHTML = ` + + خطأ بالخادم + `; + } + + document.getElementById('latency').innerHTML = `${latency} ms`; + document.getElementById('requests').innerHTML = requestCount.toLocaleString(); + + } catch (error) { + document.getElementById('status').innerHTML = ` + + غير متصل + `; + document.getElementById('latency').innerHTML = `- ms`; + } +} + +setInterval(checkHealth, 5000); +checkHealth(); diff --git a/dashboard/style.css b/dashboard/style.css new file mode 100644 index 0000000..ec53c9b --- /dev/null +++ b/dashboard/style.css @@ -0,0 +1,123 @@ +@import url('https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;600;700&display=swap'); + +:root { + --bg-color: #0f172a; + --card-bg: rgba(30, 41, 59, 0.7); + --text-main: #f8fafc; + --text-muted: #94a3b8; + --accent: #3b82f6; + --success: #10b981; + --danger: #ef4444; + --glow-success: rgba(16, 185, 129, 0.4); + --glow-danger: rgba(239, 68, 68, 0.4); +} + +body { + font-family: 'Outfit', sans-serif; + background-color: var(--bg-color); + background-image: + radial-gradient(at 0% 0%, rgba(59, 130, 246, 0.15) 0px, transparent 50%), + radial-gradient(at 100% 100%, rgba(139, 92, 246, 0.15) 0px, transparent 50%); + color: var(--text-main); + text-align: center; + min-height: 100vh; + margin: 0; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: 20px; +} + +h1 { + font-size: 3rem; + font-weight: 700; + margin-bottom: 40px; + background: linear-gradient(to right, #60a5fa, #c084fc); + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; + letter-spacing: 2px; +} + +.dashboard-container { + display: flex; + gap: 20px; + flex-wrap: wrap; + justify-content: center; + max-width: 900px; +} + +.card { + background: var(--card-bg); + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); + border: 1px solid rgba(255, 255, 255, 0.1); + padding: 30px; + border-radius: 20px; + min-width: 250px; + box-shadow: 0 10px 30px -10px rgba(0, 0, 0, 0.5); + transition: transform 0.3s ease, box-shadow 0.3s ease; + position: relative; + overflow: hidden; +} + +.card:hover { + transform: translateY(-5px); + box-shadow: 0 20px 40px -10px rgba(0, 0, 0, 0.6); + border: 1px solid rgba(255, 255, 255, 0.2); +} + +.card::before { + content: ''; + position: absolute; + top: 0; left: 0; right: 0; height: 3px; + background: linear-gradient(to right, transparent, var(--accent), transparent); + opacity: 0.5; +} + +.card-title { + font-size: 1.1rem; + color: var(--text-muted); + margin-bottom: 15px; + text-transform: uppercase; + letter-spacing: 1px; +} + +.card-value { + font-size: 2.5rem; + font-weight: 600; + display: flex; + align-items: center; + justify-content: center; + gap: 10px; +} + +.good { + color: var(--success); + text-shadow: 0 0 15px var(--glow-success); +} +.bad { + color: var(--danger); + text-shadow: 0 0 15px var(--glow-danger); +} +.unknown { color: var(--text-muted); } + +/* Ping Animation for Status */ +.status-indicator { + display: inline-block; + width: 15px; height: 15px; + border-radius: 50%; + position: relative; +} +.status-indicator.good { background-color: var(--success); } +.status-indicator.bad { background-color: var(--danger); } +.status-indicator.good::after { + content: ''; + position: absolute; width: 100%; height: 100%; top: 0; left: 0; + border-radius: 50%; background-color: var(--success); + animation: ping 2s cubic-bezier(0, 0, 0.2, 1) infinite; +} + +@keyframes ping { + 75%, 100% { transform: scale(2.5); opacity: 0; } +} diff --git a/mithal_monitor/dashboard/index.html b/mithal_monitor/dashboard/index.html new file mode 100644 index 0000000..a399613 --- /dev/null +++ b/mithal_monitor/dashboard/index.html @@ -0,0 +1,163 @@ + + + + + Mithal.space Monitoring + + + + + + +

لوحة مراقبة Mithal.space 🚀

+ +
+
+
نسبة التوافر (Uptime)
+
99.9 %
+
+
+
صلاحية شهادة SSL
+
64 يوم متبقي
+
+
+
متوسط الاستجابة
+
120 ms
+
+
+ +
+
زمن الاستجابة (آخر 60 دقيقة)
+ +
+ +
+
+
سجل الفحوصات (آخر 10)
+ + + + + + + + + + + + + +
الوقتالحالةالاستجابة (ms)DNS (ms)بحث (ms)
جاري جلب البيانات...
+
+
+ + + + diff --git a/mithal_monitor/deploy_instructions.md b/mithal_monitor/deploy_instructions.md new file mode 100644 index 0000000..64b4e09 --- /dev/null +++ b/mithal_monitor/deploy_instructions.md @@ -0,0 +1,39 @@ +# Deployment Instructions: Mithal Monitoring Dashboard + +To serve the HTML dashboard and the collected `metrics.json` data, we will deploy a simple Nginx container to ghaymah.systems. + +## 1. Directory Structure Setup +Move the dashboard HTML and the `metrics.json` file into a single directory to be served: +```bash +mkdir -p /Users/mac/a1/mithal_monitor/deploy/public +cp /Users/mac/a1/mithal_monitor/dashboard/index.html /Users/mac/a1/mithal_monitor/deploy/public/ +# Note: The monitor.py script should be configured to write metrics.json to this 'public' folder. +``` + +## 2. Nginx Dockerfile +Create a `Dockerfile` in `/Users/mac/a1/mithal_monitor/deploy/`: +```dockerfile +FROM nginx:alpine +COPY public/ /usr/share/nginx/html/ +EXPOSE 80 +``` + +## 3. Deployment Steps +Using the `ghaymah` CLI: + +```bash +cd /Users/mac/a1/mithal_monitor/deploy + +# Build and Push +docker build -t registry.ghaymah.systems/my-org/mithal-dashboard:latest . +docker push registry.ghaymah.systems/my-org/mithal-dashboard:latest + +# Deploy +ghaymah deploy \ + --name mithal-dashboard \ + --image registry.ghaymah.systems/my-org/mithal-dashboard:latest \ + --port 80 \ + --env production +``` + +The dashboard will now be accessible via the URL provided by ghaymah.systems, and it will serve `index.html` as well as `metrics.json` over HTTP(S). diff --git a/mithal_monitor/monitor.py b/mithal_monitor/monitor.py new file mode 100644 index 0000000..95f4353 --- /dev/null +++ b/mithal_monitor/monitor.py @@ -0,0 +1,95 @@ +import requests +import time +import socket +import ssl +import json +import os +from datetime import datetime +import dns.resolver + +TARGET_URL = "https://mithal.space" +TARGET_DOMAIN = "mithal.space" +DATA_FILE = "/Users/mac/a1/mithal_monitor/data/metrics.json" + +def check_ssl(domain): + context = ssl.create_default_context() + try: + with socket.create_connection((domain, 443), timeout=5) as sock: + with context.wrap_socket(sock, server_hostname=domain) as ssock: + cert = ssock.getpeercert() + expiry_date = datetime.strptime(cert['notAfter'], "%b %d %H:%M:%S %Y %Z") + days_remaining = (expiry_date - datetime.utcnow()).days + return "Valid", days_remaining + except Exception as e: + return f"Error: {str(e)}", 0 + +def check_dns(domain): + start = time.time() + try: + answers = dns.resolver.resolve(domain, 'A') + return round((time.time() - start) * 1000, 2) + except Exception: + return -1 + +def measure(): + results = { + "timestamp": datetime.utcnow().isoformat() + "Z", + "latency_ms": -1, + "status_code": 0, + "uptime": False, + "ssl_status": "Unknown", + "ssl_days": 0, + "dns_time_ms": -1, + "search_time_ms": -1 + } + + # DNS Check + results["dns_time_ms"] = check_dns(TARGET_DOMAIN) + + # SSL Check + ssl_status, ssl_days = check_ssl(TARGET_DOMAIN) + results["ssl_status"] = ssl_status + results["ssl_days"] = ssl_days + + # HTTP Check + try: + start = time.time() + resp = requests.get(TARGET_URL, timeout=5) + results["latency_ms"] = round((time.time() - start) * 1000, 2) + results["status_code"] = resp.status_code + results["uptime"] = resp.status_code < 400 + except Exception: + pass + + # Mock Search Check (assuming a /search endpoint exists) + try: + start = time.time() + requests.get(f"{TARGET_URL}/search?q=test", timeout=5) + results["search_time_ms"] = round((time.time() - start) * 1000, 2) + except Exception: + pass + + return results + +def save_data(data): + os.makedirs(os.path.dirname(DATA_FILE), exist_ok=True) + history = [] + if os.path.exists(DATA_FILE): + try: + with open(DATA_FILE, 'r') as f: + history = json.load(f) + except json.JSONDecodeError: + pass + + history.append(data) + # Keep last 1440 checks (24 hours at 1/min) + history = history[-1440:] + + with open(DATA_FILE, 'w') as f: + json.dump(history, f, indent=2) + +if __name__ == "__main__": + while True: + data = measure() + save_data(data) + time.sleep(60) diff --git a/monitor/monitor.sh b/monitor/monitor.sh new file mode 100755 index 0000000..fcff5ed --- /dev/null +++ b/monitor/monitor.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +LOG_FILE="/Users/mac/a1/monitor/monitor.log" + +# Ensure log file exists +mkdir -p "$(dirname "$LOG_FILE")" +: > "$LOG_FILE" + +while true; do + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + RESPONSE=$(curl -s -o /dev/null -w "%{http_code} %{time_total}" http://localhost:8080/health) + echo "$TIMESTAMP $RESPONSE" >> "$LOG_FILE" + sleep 30 +done diff --git a/postmortem/postmortem.md b/postmortem/postmortem.md new file mode 100644 index 0000000..e06e4af --- /dev/null +++ b/postmortem/postmortem.md @@ -0,0 +1,28 @@ +# Incident Post-mortem: API Service OOMKilled Outage + +## 1. Summary +- **Date & Time:** [Insert Date] +- **Duration:** 45 minutes +- **Impact:** API service was completely unavailable for users, resulting in a 100% error rate (502 Bad Gateway / 503 Service Unavailable) during the incident window. +- **Root Cause:** Container memory limit was exceeded, causing the Kubernetes/Cloud orchestrator to continuously terminate the pod with an `OOMKilled` status. + +## 2. Timeline (UTC) +- **10:00 AM:** Monitoring alerts triggered for high error rates on the `/health` endpoint. +- **10:05 AM:** On-call engineer acknowledged the alert and started investigation. +- **10:15 AM:** Engineer identified that the container was crash-looping with `OOMKilled` exit code 137. +- **10:25 AM:** A temporary mitigation was applied by manually increasing the container memory limit from 512MB to 1024MB. +- **10:35 AM:** Service stabilized. Containers remained running without restarts. +- **10:45 AM:** Incident marked resolved after 10 minutes of stable metrics. + +## 3. Root Cause Analysis (The "5 Whys") +1. **Why did the service go down?** The container was repeatedly killed by the orchestrator. +2. **Why was it killed?** The orchestrator issued an `OOMKilled` (Out Of Memory) signal. +3. **Why did it run out of memory?** The application consumed more memory than its allocated limit (512MB). +4. **Why did it consume so much memory?** An unexpected spike in requests (or a memory leak in a newly deployed feature) caused the application stack to load massive objects into memory simultaneously. +5. **Why wasn't this caught or handled?** The auto-scaling policy was based solely on CPU, so it didn't spin up new instances to distribute the memory load. + +## 4. Recommendations & Action Items +- **Immediate:** Keep the memory limit at 1024MB until a thorough memory profiling is completed. +- **Short-term:** Implement a memory-based auto-scaling rule (Scale out when Memory > 70%). +- **Medium-term:** Setup early-detection alerts for memory utilization reaching 80% to warn the team *before* an OOMKilled event occurs. +- **Long-term:** Profile the application to identify memory bottlenecks or leaks. diff --git a/scaling/auto_scaling_policy.md b/scaling/auto_scaling_policy.md new file mode 100644 index 0000000..e5ae81a --- /dev/null +++ b/scaling/auto_scaling_policy.md @@ -0,0 +1,25 @@ +# Auto-Scaling Policy for ghaymah.systems + +To prevent repeating the OOMKilled outage, the platform's auto-scaling group (ASG) must be configured to respond to memory pressure as well as CPU load. + +## 1. Scale-Out Policy (Adding Instances) +- **Metric:** Average Container Memory Utilization +- **Threshold:** > 70% +- **Evaluation Period:** 2 minutes (2 consecutive data points of 1 minute each) +- **Action:** Add 1 container instance (Step scaling) or scale by 20% of current capacity. +- **Cooldown Period:** 3 minutes (allows the new container to boot and start serving traffic before evaluating again). + +## 2. Scale-In Policy (Removing Instances) +- **Metric:** Average Container Memory Utilization +- **Threshold:** < 40% +- **Evaluation Period:** 5 minutes +- **Action:** Remove 1 container instance. +- **Cooldown Period:** 5 minutes (prevents aggressive scale-in which might cause immediate resource pressure). + +## 3. CPU Backup Policy +*Maintain existing CPU policies as a secondary trigger:* +- Scale out if Average CPU > 75% for 2 minutes. + +## 4. Minimum / Maximum Capacity +- **Min Containers:** 2 (for high availability across zones) +- **Max Containers:** 20 (to control billing, can be adjusted based on anticipated load) diff --git a/scaling/early_detection.md b/scaling/early_detection.md new file mode 100644 index 0000000..94ab6bd --- /dev/null +++ b/scaling/early_detection.md @@ -0,0 +1,25 @@ +# Early Detection of Memory Issues + +Waiting for an application to crash (OOMKilled) is a reactive approach. To proactively detect memory issues, we must configure our monitoring tools (Prometheus, Datadog, or ghaymah metrics). + +## 1. High-Watermark Alerting +Configure alerts on the metric `container_memory_usage_bytes` (or equivalent). + +- **Warning Alert (Slack/Teams):** + - Trigger: Container Memory > 80% of limit + - Duration: Sustained for > 3 minutes. + - Action: Alerts the engineering team during business hours to investigate potential memory leaks. + +- **Critical Alert (PagerDuty/Phone Call):** + - Trigger: Container Memory > 90% of limit + - Duration: Sustained for > 2 minutes. + - Action: Wakes up the on-call engineer to apply mitigations (e.g., manual scaling, restarting pods) before the crash happens. + +## 2. Rate of Change Alerting (Anomaly Detection) +Sometimes memory doesn't hit a static threshold, but it grows unusually fast. +- Monitor the *derivative* (rate of change) of memory usage. +- If memory grows by more than 20% within 5 minutes (without a corresponding 20% spike in traffic), trigger an anomaly alert. + +## 3. APM Profiling +- Integrate APM (Application Performance Monitoring) to track Garbage Collection (GC) pauses in languages like Java/Node.js, or memory footprint per request in Python/Go. +- A sudden increase in GC time is often a precursor to an OOM event.