commit 90256c2ab89fab832f82b86da7a0ed318399eb96 Author: ahmed Date: Tue Jul 28 15:37:07 2026 +0300 feat: initialize repository with SRE infrastructure, monitoring dashboard, and deployment documentation diff --git a/README.md b/README.md new file mode 100644 index 0000000..fab4ad7 --- /dev/null +++ b/README.md @@ -0,0 +1,230 @@ +# 🚀 Ghaymah SRE Practical Exam + +**Candidate Name:** Ahmed Abdelaziz Hussein +**Track:** Site Reliability Engineering (SRE) + +--- + +## 📁 Repository Structure + +Below is the directory tree of the submission, showing all implementation files: + +```text +. +├── common-qabilah +│ └── qabilah-profile.txt +├── q1-deploy-monitor +│ ├── Dockerfile +│ ├── dashboard.html +│ ├── health-check.sh +│ └── status.json +├── q2-postmortem +│ └── postmortem-report.md +├── q3-cicd +│ └── workflow.yml +├── q4-sacalability +│ ├── architecture.png +│ └── calculations.md +└── q5-mithal-monitor + ├── dashbourd.html + ├── metrics.json + └── monitor.py +``` + +--- + +## 🔧 Technologies Used + +The project leverages the following technologies and frameworks: +- **Go / Fiber** (API Development) +- **Docker** (Containerization & Multi-stage builds) +- **GitHub Actions** (CI/CD Pipeline) +- **HTML / CSS / JavaScript** (Dashboards & Visualization) +- **Python** (Mithal Automated Monitoring) +- **Bash** (Monitoring Agent scripts) +- **Ghaymah Cloud** (Deployment Infrastructure & CLI) + +--- + +## 📊 Question 1 – Deploy Application & Monitoring + +### Requirements Covered +| Requirement | Status | Details | +| :--- | :---: | :--- | +| Dockerfile | ✔ | Optimized multi-stage build starting from `golang:alpine` to `scratch` | +| Simple Go Fiber API | ✔ | REST API utilizing the Go Fiber framework | +| `/health` endpoint | ✔ | Returns application health status and current timestamp | +| `/metrics` endpoint | ✔ | Exposes real-time internal metrics (uptime, request count, latency) | +| Monitoring Script | ✔ | Bash script performing periodic checks and exporting metrics to JSON | +| Monitoring Dashboard | ✔ | Static HTML dashboard to display real-time statuses and metrics | +| Deployment on Ghaymah Cloud | ✔ | API deployed live on Ghaymah systems | + +### Files +- [`q1-deploy-monitor/Dockerfile`](./q1-deploy-monitor/Dockerfile) +- [`q1-deploy-monitor/health-check.sh`](./q1-deploy-monitor/health-check.sh) +- [`q1-deploy-monitor/dashboard.html`](./q1-deploy-monitor/dashboard.html) +- [`q1-deploy-monitor/status.json`](./q1-deploy-monitor/status.json) + +### Proof of Implementation +The screenshots showcasing the running state are located under `docs/screenshots/`: + +1. **Deployment on Ghaymah Cloud** + ![Deployment on Ghaymah Cloud](docs/screenshots/ghaymah-deployment.png) + +2. **Monitoring Script Running** + ![Monitoring Script Running](docs/screenshots/monitoring-script.png) + +3. **Dashboard** + ![Monitoring Dashboard](docs/screenshots/dashboard.png) +--- + +## 📑 Question 2 – Incident Postmortem + +### Requirements Covered +- [x] **Executive Summary** (Incident metadata, downtime duration, root cause, impact) +- [x] **Timeline** (Detailed chronological sequence of events from spike to resolution) +- [x] **Root Cause Analysis (RCA)** (Detailed diagnosis of resource limits and missing HPA) +- [x] **Recommendations** (Immediate P0 fixes and long-term action items) +- [x] **Auto Scaling Policy** (Configured Horizontal Pod Autoscaler YAML config for Ghanimah) +- [x] **Monitoring Strategy** (Prometheus alert rules for early warning and OOM checks) + +### Files +- [`q2-postmortem/postmortem-report.md`](file:///home/ahmed/ghaymah_task/ghaymah-exam-ahmed-abdelaziz-SRE/q2-postmortem/postmortem-report.md) + +--- + +## ⚙️ Question 3 – CI/CD Pipeline + +### Requirements Covered +- [x] **Docker Build:** Builds container images on triggers. +- [x] **Container Registry:** Image tagging for version control. +- [x] **Deploy to Staging:** Triggered automatically on release branches. +- [x] **Manual Approval:** Gatekeeping promotion using GitHub Environments rules. +- [x] **Deploy to Production:** Triggered on main branch post-approval. +- [x] **Ghaymah CLI Integration:** Automated login and app launch in workflow jobs. + +### Deployment Flow Diagram +```text +Push + ↓ +Build + ↓ +Deploy Staging + ↓ +Manual Approval + ↓ +Deploy Production +``` + +### Files +- [`q3-cicd/workflow.yml`](./q3-cicd/workflow.yml) + +### Proof of Implementation +1. **Staging Deployment** + ![Staging Deployment](docs/screenshots/staging-deployment.png) + +2. **Manual Approval** + ![Manual Approval](docs/screenshots/manual-approval.png) + +3. **Production Deployment** + ![Production Deployment](docs/screenshots/production-environment.png) + + +--- + +## 📈 Question 4 – Scalability & Load Balancing + +### Requirements Covered +- [x] **Architecture Diagram** (Visual flow of requests to handle 15,000 req/s across zones) +- [x] **Capacity Calculation** (Calculations justifying 39 Pods based on limits and 30% safety buffer) +- [x] **Cold Start Strategy** (Multi-stage scratch image design, pre-warming, and probe tuning) +- [x] **Block Storage** (Dynamic PV/PVC configurations utilizing Ghanimah Block Storage NVMe disks) + +### Files +- [`q4-sacalability/calculations.md`](./q4-sacalability/calculations.md) +- [`q4-sacalability/architecture.png`](./q4-sacalability/architecture.png) + +--- + +## 🖥️ Question 5 – Mithal Monitoring Dashboard + +### Requirements Covered +- [x] **Latency Monitoring:** Measures and logs home page loading speeds. +- [x] **Uptime Monitoring:** Tracks availability and monitors return status codes. +- [x] **SSL Monitoring:** Calculates certificate expiration and counts remaining days. +- [x] **DNS Monitoring:** Resolves and logs lookup times. +- [x] **Search Response Monitoring:** Assesses specific query parameters response time. +- [x] **JSON Storage:** Logs state to dynamic history files up to 24 hours. +- [x] **Dashboard:** An interface displaying key charts and status panels. +- [x] **Deployment on Ghaymah:** Live monitor configuration on Ghaymah infrastructure. + +### Files +- [`q5-mithal-monitor/monitor.py`](file:///home/ahmed/ghaymah_task/ghaymah-exam-ahmed-abdelaziz-SRE/q5-mithal-monitor/monitor.py) +- [`q5-mithal-monitor/metrics.json`](file:///home/ahmed/ghaymah_task/ghaymah-exam-ahmed-abdelaziz-SRE/q5-mithal-monitor/metrics.json) +- [`q5-mithal-monitor/dashbourd.html`](file:///home/ahmed/ghaymah_task/ghaymah-exam-ahmed-abdelaziz-SRE/q5-mithal-monitor/dashbourd.html) + +### Proof of Implementation +1. **monitor.py Running** + ![monitor.py Running](docs/screenshots/monitor-running.png) + +2. **Dashboard** + ![Mithal Dashboard](docs/screenshots/mithal-dashboard.png) +--- + +## 🏃 Running Locally + +### Question 1 – Go API & Monitoring Agent +To test the Go Fiber API and the monitoring scripts locally: + +1. **Build and Run the Go API via Docker:** + ```bash + # Build the container image + docker build -t ghaymah-sre-api ./q1-deploy-monitor + + # Run the container exposing port 8080 + docker run -d -p 8080:8080 --name ghanimah-api ghaymah-sre-api + ``` +2. **Run the Bash Monitoring Agent:** + ```bash + # Point the agent to the locally running Docker container + API_URL=http://localhost:8080 bash q1-deploy-monitor/health-check.sh + ``` +3. **View the Dashboard:** + Open [`q1-deploy-monitor/dashboard.html`](./q1-deploy-monitor/dashboard.html) directly in a web browser of your choice. + +--- + +### Question 5 – Mithal Automated Monitor +To test the Python monitoring daemon locally: + +1. **Install Dependencies:** + ```bash + pip install requests + ``` +2. **Run the Monitor Script:** + ```bash + python3 q5-mithal-monitor/monitor.py + ``` +3. **View the Mithal Dashboard:** + Open [`q5-mithal-monitor/dashbourd.html`](./q5-mithal-monitor/dashbourd.html) in your browser to view historical metrics. + +--- + +## ✨ Project Highlights + +This repository demonstrates complete implementation of fundamental Site Reliability Engineering practices: +- **Containerization:** Clean multi-stage lightweight builds (Go statically-linked binary in a `scratch` container). +- **Monitoring & Observability:** Real-time metrics gathering, system state logging, alerts modeling, and custom front-end status dashboards. +- **CI/CD:** Automated builds, environments targeting, manual approvals, and deployment orchestrations. +- **Incident Analysis:** Professional blameless postmortem report detailing timelines, root cause analysis, action items, auto-scaling thresholds, and early discovery strategies. +- **Scalability:** Quantitative capacity sizing for high-traffic environments (15,000 req/s), Cold Start tuning, and stateful volume management. +- **Cloud Deployment:** Orchestration using the Ghaymah Cloud platforms. + +--- + +## ✍️ Author + +**Ahmed Abdelaziz Hussein** +*Information Systems* +*Faculty of Computers and Information* +*Qabilah Profile:* [ahmed-abdelaziz-89943a271](https://qabilah.com/profile/ahmed-abdelaziz-89943a271/posts) diff --git a/common-qabilah/qabilah-profile.txt b/common-qabilah/qabilah-profile.txt new file mode 100644 index 0000000..a4ee94d --- /dev/null +++ b/common-qabilah/qabilah-profile.txt @@ -0,0 +1 @@ +https://qabilah.com/profile/ahmed-abdelaziz-89943a271/posts \ No newline at end of file diff --git a/docs/screenshots/app-running.png b/docs/screenshots/app-running.png new file mode 100644 index 0000000..262591a Binary files /dev/null and b/docs/screenshots/app-running.png differ diff --git a/docs/screenshots/dashboard.png b/docs/screenshots/dashboard.png new file mode 100644 index 0000000..f87a864 Binary files /dev/null and b/docs/screenshots/dashboard.png differ diff --git a/docs/screenshots/manual-approval.png b/docs/screenshots/manual-approval.png new file mode 100644 index 0000000..5085bbe Binary files /dev/null and b/docs/screenshots/manual-approval.png differ diff --git a/docs/screenshots/mithal-dashboard.png b/docs/screenshots/mithal-dashboard.png new file mode 100644 index 0000000..b3b49fd Binary files /dev/null and b/docs/screenshots/mithal-dashboard.png differ diff --git a/docs/screenshots/monitor-running.png b/docs/screenshots/monitor-running.png new file mode 100644 index 0000000..cec5b53 Binary files /dev/null and b/docs/screenshots/monitor-running.png differ diff --git a/docs/screenshots/monitoring-script.png b/docs/screenshots/monitoring-script.png new file mode 100644 index 0000000..b3e5648 Binary files /dev/null and b/docs/screenshots/monitoring-script.png differ diff --git a/docs/screenshots/staging-deployment.png b/docs/screenshots/staging-deployment.png new file mode 100644 index 0000000..3f6dcca Binary files /dev/null and b/docs/screenshots/staging-deployment.png differ diff --git a/q1-deploy-monitor/Dockerfile b/q1-deploy-monitor/Dockerfile new file mode 100644 index 0000000..45e4351 --- /dev/null +++ b/q1-deploy-monitor/Dockerfile @@ -0,0 +1,32 @@ +# Stage 1: Build binary using lightweight Go Alpine image +FROM golang:1.24-alpine AS builder + +WORKDIR /app + +# Cache dependencies +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Build lightweight, statically linked binary stripped of debug information (-s -w) +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o server main.go + +# Stage 2: Minimal runtime image using scratch (~10-15MB final image size) +FROM scratch + +# Copy CA certificates for HTTPS requests if needed +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + +# Copy static binary from builder stage +COPY --from=builder /app/server /server + +# Run as non-root user (nobody) for enhanced security +USER 65534:65534 + +# Expose port 8080 +EXPOSE 8080 + +# Launch server +ENTRYPOINT ["/server"] diff --git a/q1-deploy-monitor/dashboard.html b/q1-deploy-monitor/dashboard.html new file mode 100644 index 0000000..f1f566b --- /dev/null +++ b/q1-deploy-monitor/dashboard.html @@ -0,0 +1,411 @@ + + + + + + Ghaymah SRE - لوحة المراقبة + + + + + + +
+ +
+
+

🌩️ Ghaymah SRE Monitor

+

مراقبة أداء واستجابة التطبيق المباشرة (Real-time Health Dashboard)

+
+
+ + مباشر (تحديث كل 3 ثوانٍ) +
+
+ + +
+ +
+
+ حالة النظام (Status) +
⚡
+
+
+ ● متصل (Healthy) +
+
كود الاستجابة: HTTP 200 OK
+
+ + +
+
+ زمن الاستجابة (Latency) +
⏱️
+
+
-- ms
+
+ متوسط الاستجابة: + -- ms +
+
+ + +
+
+ إجمالي الطلبات (Total Requests) +
📊
+
+
--
+
+ مدة التشغيل (Uptime): + -- ثانية +
+
+
+ + +
+
+

سجل الفحوصات الأخيرة (Health Check Logs)

+ الهدف: http://localhost:8080/metrics +
+ + + + + + + + + + + + + + + +
الوقت (UTC)الحالةكود HTTPزمن الاستجابةإجمالي الطلبات
جاري جلب البيانات...
+
+
+ + + + diff --git a/q1-deploy-monitor/health-check.sh b/q1-deploy-monitor/health-check.sh new file mode 100755 index 0000000..8bdc8b3 --- /dev/null +++ b/q1-deploy-monitor/health-check.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Configuration +API_URL="${API_URL:-https://ghaymah-task-da46b224b9fc.hosted.ghaymah.systems}" +CHECK_INTERVAL=30 +STATUS_FILE="status.json" + +echo "==================================================" +echo " Starting Ghaymah SRE Monitoring Agent " +echo " Target URL : $API_URL " +echo " Interval : ${CHECK_INTERVAL}s " +echo "==================================================" + +while true; do + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + START_TIME=$(date +%s%N) + HTTP_RESPONSE=$(curl -s -w "\n%{http_code}" "${API_URL}/metrics") + END_TIME=$(date +%s%N) + + # Calculate latency in milliseconds + LATENCY_MS=$(( (END_TIME - START_TIME) / 1000000 )) + + HTTP_BODY=$(echo "$HTTP_RESPONSE" | sed '$d') + HTTP_CODE=$(echo "$HTTP_RESPONSE" | tail -n1) + + if [ "$HTTP_CODE" -eq 200 ]; then + STATUS="healthy" + UPTIME=$(echo "$HTTP_BODY" | grep -o '"uptime_seconds":[0-9]*' | cut -d':' -f2) + TOTAL_REQ=$(echo "$HTTP_BODY" | grep -o '"total_requests":[0-9]*' | cut -d':' -f2) + AVG_LATENCY=$(echo "$HTTP_BODY" | grep -o '"average_latency_ms":[0-9.]*' | cut -d':' -f2) + LAST_LATENCY=$(echo "$HTTP_BODY" | grep -o '"last_request_latency_ms":[0-9.]*' | cut -d':' -f2) + + echo "[ $TIMESTAMP ] [OK 200] Status: $STATUS | Latency: ${LATENCY_MS}ms | Total Requests: ${TOTAL_REQ:-0} | Avg Latency: ${AVG_LATENCY:-0}ms" + + # Export JSON metrics status file for dashboard consumption + cat < "$STATUS_FILE" +{ + "status": "$STATUS", + "http_code": $HTTP_CODE, + "check_timestamp": "$TIMESTAMP", + "check_latency_ms": $LATENCY_MS, + "uptime_seconds": ${UPTIME:-0}, + "total_requests": ${TOTAL_REQ:-0}, + "average_latency_ms": ${AVG_LATENCY:-0}, + "last_request_latency_ms": ${LAST_LATENCY:-$LATENCY_MS} +} +EOF + else + STATUS="unhealthy" + echo "[ $TIMESTAMP ] [ERROR ${HTTP_CODE:-000}] Target $API_URL is DOWN or unreachable! Latency: ${LATENCY_MS}ms" + + cat < "$STATUS_FILE" +{ + "status": "$STATUS", + "http_code": ${HTTP_CODE:-0}, + "check_timestamp": "$TIMESTAMP", + "check_latency_ms": $LATENCY_MS, + "uptime_seconds": 0, + "total_requests": 0, + "average_latency_ms": 0, + "last_request_latency_ms": 0 +} +EOF + fi + + sleep $CHECK_INTERVAL +done diff --git a/q1-deploy-monitor/status.json b/q1-deploy-monitor/status.json new file mode 100644 index 0000000..58cfee8 --- /dev/null +++ b/q1-deploy-monitor/status.json @@ -0,0 +1,10 @@ +{ + "status": "healthy", + "http_code": 200, + "check_timestamp": "2026-07-28T10:09:20Z", + "check_latency_ms": 67, + "uptime_seconds": 5973, + "total_requests": 203, + "average_latency_ms": 1.76, + "last_request_latency_ms": 0.27 +} diff --git a/q2-postmortem/postmortem-report.md b/q2-postmortem/postmortem-report.md new file mode 100644 index 0000000..270f65e --- /dev/null +++ b/q2-postmortem/postmortem-report.md @@ -0,0 +1,146 @@ +# 📋 تقرير تحليل حادثة انقطاع الخدمة — Postmortem Report + +--- + +## 1️⃣ تقرير الـ Postmortem التفصيلي + +### 📑 الملخص التنفيذي (Executive Summary) +* **اسم الحادثة:** INC-2026-0728-OOM +* **تاريخ الحادثة:** 28 يوليو 2026 +* **مدة الانقطاع (Downtime):** 45 دقيقة (10:00 - 10:45 UTC) +* **مستوى الأهمية (Severity):** Critical (SEV-1) +* **السبب الرئيسي:** حدوث حالات `OOMKilled` (Out Of Memory) متكررة للحاويات نتيجة ارتفاع مفاجئ في حركة المرور (Traffic Spike) مع عدم وجود سياسة للتوسع التلقائي (Auto-scaling) وضيق حدود الذاكرة المخصصة. +* **الأثر على الخدمة:** توقف تام لأحد الخدمات الأساسية (HTTP 502 Bad Gateway) وتضرر 100% من طلبات المستخدمين أثناء فترة الانقطاع. + +--- + +### ⏱️ الجدول الزمني للحادثة (Timeline of Events) + +| الوقت (UTC) | الحدث | +| :--- | :--- | +| **10:00** | بدء ارتفاع مفاجئ في عدد الطلبات الموجهة للخدمة (تضاعف عدد الطلبات 5 مرات). | +| **10:05** | تجاوز استهلاك الذاكرة الحد الأقصى المخصص (`Memory Limit = 512MiB`)، وقام الـ Linux Kernel بإنهاء الحاوية (`OOMKilled - Exit Code 137`). | +| **10:08** | دخول الحاوية في حالة `CrashLoopBackOff` بسبب تكرار الـ OOM فور إعادة التشغيل. | +| **10:12** | انطلاق أول تنبيه (Health Check Failure) وفشلت فحوصات الجاهزية (Liveness & Readiness Probes). | +| **10:20** | استجابة فريق الـ SRE وبدء التحقيق في المشكلة عبر مراجعة السجلات والتنبيهات. | +| **10:30** | تحديد السبب الجذر: حدوث `OOMKilled` متكرر لجميع الـ Replicas الشغالة. | +| **10:38** | **إجراء طارئ:** رفع حدود الذاكرة يدويًا من `512MiB` إلى `2GiB` وزيادة عدد الـ Pods من 2 إلى 6. | +| **10:45** | استقرار جميع الحاويات، وعودة مؤشرات الخدمة إلى الوضع الطبيعي (HTTP 200 OK) وانتهاء الحادثة. | + +--- + +### 🔍 السبب الجذر (Root Cause Analysis - RCA) + +1. **قصور في تخصيص الموارد (Insufficient Memory Limits):** + * تم تخصيص حد ذاكرة منخفض جِدًّا (`512MiB`) لا يتناسب مع حجم العمل المرتفع، مما جعل التطبيق عرضة للإنهاء المباشر بواسطة الـ OOM Killer عند حدوث ضغط. +2. **غياب التوسع التلقائي (No Auto-scaling):** + * لم يتم إعداد `HorizontalPodAutoscaler` (HPA) للتوسع الأفقي عند ارتفاع الضغط. +3. **تنبيهات متأخرة للذاكرة:** + * التنبيهات كانت مجهزة فقط عند توقف الخدمة تمامًا (`Health Check Failed`) بدلاً من التنبيه المبكر عند وصول استهلاك الذاكرة إلى 80%. + +--- + +### 🛠️ التوصيات والإجراءات التصحيحية (Action Items) + +| الرقم | الإجراء (Action Item) | النوع | الأولوية | +| :---: | :--- | :---: | :---: | +| **1** | زيادة الـ Memory Limits الأساسية من `512MiB` إلى `1GiB` وتحديد `Requests` عند `512MiB`. | فوري (Immediate) | 🔴 P0 | +| **2** | تطبيق سياسة Auto-scaling أفقية (HPA) لمنصة غنيمة تعمل على الذاكرة والـ CPU. | متوسط (Medium) | 🔴 P0 | +| **3** | إضافة قواعد تنبيه مبكرة عند تجاوز الذاكرة نسبة 80% وقبل الوصول لـ 100%. | فوري (Immediate) | 🟡 P1 | +| **4** | إجبار إجراء اختبارات ضغط (Load & Stress Testing) قبل الترقية للإنتاج. | طويل المدى | 🟢 P2 | + +--- + +## 2️⃣ تصميم سياسة Auto-scaling لمنصة غنيمة (Ghanimah Auto-scaling Policy) + +لتفادي تكرار حادثة `OOMKilled` مستقبلاً، تم تصميم سياسة **Horizontal Pod Autoscaler (HPA)** مخصصة لمنصة **غنيمة**: + +### 📐 المكونات والحدود (Policy Configuration) + +```yaml +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: ghanimah-app-hpa + namespace: production +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: ghanimah-sre-api + minReplicas: 3 # الحد الأدنى لضمان العزل والتوافر العالي + maxReplicas: 15 # الحد الأقصى للاستجابة للهجمات أو الضغط العالي + metrics: + # 1. التوسع بناءً على الذاكرة (Memory Utilization) - منع الـ OOM + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 70 # التوسع فور وصول الذاكرة إلى 70% + # 2. التوسع بناءً على المعالج (CPU Utilization) + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 75 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 # توسع فوري (0 ثانية) لتجنب OOMKilled + policies: + - type: Percent + value: 100 # مضاعفة عدد الـ Pods فوراً عند الضغط + periodSeconds: 15 + scaleDown: + stabilizationWindowSeconds: 300 # الانتظار 5 دقائق قبل تقليص العدد لمنع Flapping +``` + +### 💡 قواعد الموارد في الحاوية (Resource Requests & Limits) +```yaml +resources: + requests: + cpu: "250m" + memory: "512Mi" + limits: + cpu: "1000m" + memory: "1024Mi" +``` + +--- + +## 3️⃣ الكشف المبكر والمراقبة باستخدام أدوات غنيمة (Early Detection & Observability) + +لكشف مشكلة الذاكرة قبل وصولها لمرحلة `OOMKilled`: + +### 1. إعداد قواعد التنبيه المبكر (Prometheus Alert Rules) + +* **تنبيه تحذيري لارتفاع الذاكرة (Memory Usage Warning > 80%):** + ```yaml + alert: HighMemoryUsageWarning + expr: (container_memory_working_set_bytes{container!=""} / container_spec_memory_limit_bytes{container!=""}) * 100 > 80 + for: 2m + labels: + severity: warning + annotations: + summary: "استهلاك الذاكرة تجاوز 80% في الحاوية {{ $labels.pod }}" + ``` + +* **تنبيه عاجل لحدث OOMKilled أو إعادة تشغيل متكررة:** + ```yaml + alert: ContainerOOMKilledDetected + expr: increase(kube_pod_container_status_restarts_total[5m]) > 2 + for: 0m + labels: + severity: critical + annotations: + summary: "تم اكتشاف إعادة تشغيل متكررة للحاوية {{ $labels.pod }} - احتمال OOMKilled" + ``` + +--- + +### 2. أهم المؤشرات في لوحة المراقبة (Dashboard Metrics) + +1. **`container_memory_working_set_bytes`**: قياس حجم الذاكرة المستخدمة فعلياً مقارنة بالحد الأقصى (`container_spec_memory_limit_bytes`). +2. **`kube_pod_container_status_last_terminated_reason`**: الكشف الفوري عن قيمة `OOMKilled`. +3. **`rate(http_requests_total[1m])`**: متابعة نمو الطلبات للتنبؤ بالضغط والتوسع المبكر. diff --git a/q3-cicd/workflow.yml b/q3-cicd/workflow.yml new file mode 100644 index 0000000..919eefd --- /dev/null +++ b/q3-cicd/workflow.yml @@ -0,0 +1,75 @@ +name: Ghaymah CI/CD Pipeline + +on: + push: + branches: + - "release/**" + - main + workflow_dispatch: + +jobs: + build: + name: Build Docker Image + runs-on: ubuntu-latest + + steps: + - name: Checkout Source + uses: actions/checkout@v5 + + - name: Build Docker Image + run: | + docker build -t ghaymah-sre-api:${{ github.sha }} . + + deploy-staging: + name: Deploy to Staging + needs: build + + if: startsWith(github.ref, 'refs/heads/release/') + + environment: staging + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install Ghaymah CLI + run: curl -sSL https://cli.ghaymah.systems/install.sh | bash + + - name: Login to Ghaymah + run: | + $HOME/ghaymah/bin/gy auth login \ + --email "${{ secrets.GHAYMAH_EMAIL }}" \ + --password "${{ secrets.GHAYMAH_PW }}" + + - name: Deploy to Ghaymah Staging + run: | + $HOME/ghaymah/bin/gy resource app launch + + deploy-production: + name: Deploy to Production + + needs: build + + if: github.ref == 'refs/heads/main' + + environment: + name: production + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install Ghaymah CLI + run: curl -sSL https://cli.ghaymah.systems/install.sh | bash + + - name: Login to Ghaymah + run: | + $HOME/ghaymah/bin/gy auth login \ + --email "${{ secrets.GHAYMAH_EMAIL }}" \ + --password "${{ secrets.GHAYMAH_PW }}" + + - name: Deploy to Ghaymah Production + run: | + $HOME/ghaymah/bin/gy resource app launch diff --git a/q4-sacalability/architecture.png b/q4-sacalability/architecture.png new file mode 100644 index 0000000..4b57893 Binary files /dev/null and b/q4-sacalability/architecture.png differ diff --git a/q4-sacalability/calculations.md b/q4-sacalability/calculations.md new file mode 100644 index 0000000..7e6035e --- /dev/null +++ b/q4-sacalability/calculations.md @@ -0,0 +1,110 @@ +# 🚀 قابلية التوسع وتوزيع الأحمال على منصة غنيمة (Scalability & Load Balancing) + +--- + +## 1️⃣ المخطط الهندسي للنظام (Architecture Diagram — 15,000 req/s) + +تم تصميم المعمارية التالية للاستجابة لـ **15,000 طلب في الثانية (15,000 req/s)** مع توافر عالي (High Availability) وأداء متفوق على منصة **غنيمة**: + +![المخطط الهندسي لنظام غنيمة عالي التوسع 15,000 req/s](./architecture.png) + +## 2️⃣ حساب عدد الحاويات المطلوبة (Capacity Planning & Sizing) + +### 📊 المعطيات: +* **حركة المرور المستهدفة (Target Traffic):** $15,000 \text{ req/s}$ +* **طاقة الحاوية الواحدة (Container Capacity):** $500 \text{ req/s}$ +* **هامش الأمان الموصى به (Safety Margin Buffer):** $30\%$ + +--- + +### 🧮 الخطوات الحسابية: + +1. **حساب إجمالي حركة المرور المطلوبة مع هامش الأمان:** + $$\text{Total Traffic with Buffer} = 15,000 \times (1 + 0.30) = 15,000 \times 1.30 = 19,500 \text{ req/s}$$ + +2. **حساب عدد الحاويات المطلوبة:** + $$\text{Number of Containers} = \left\lceil \frac{19,500 \text{ req/s}}{500 \text{ req/s}} \right\rceil = 39 \text{ Containers}$$ + +--- + +### 📌 النتيجة والتوزيع على بيئات غنيمة: +* **إجمالي عدد الحاويات (Pods):** **39 حاوية** (تضمن معالجة $19,500 \text{ req/s}$ بكفاءة عالية وبدون اختناق). +* **توزيع الحاويات على مناطق التوافر (Multi-AZ Deployment):** + * **Zone A:** 13 Pods + * **Zone B:** 13 Pods + * **Zone C:** 13 Pods +* **سبب إضافة هامش الأمان (30% Buffer):** + 1. امتصاص الارتفاعات المفاجئة واللحظية في حركة المرور (Traffic Spikes). + 2. تغطية استهلاك الموارد الموجه لـ Health Checks و Garbage Collection. + 3. حماية النظام أثناء إعادة تشغيل الحاويات أو تحديثات النسخ (Rolling Updates). + +--- + +## 3️⃣ استراتيجية تقليل الـ Cold Start للحاويات الجديدة + +الـ **Cold Start** هو الوقت المستغرق بين إطلاق حاوية جديدة وجاهزيتها التامة لاستقبال الطلبات. لتقليل هذا الوقت لأقل من 1 ثانية في منصة **غنيمة**، نتبع الاستراتيجيات التالية: + +### 1. تصغير حجم صورة الحاوية (Lightweight Docker Images) +* استخدام صور **Multi-stage Build** مبنية على `scratch` أو `alpine` (حجم الصورة النهائي **~6.7 ميجابايت** كما تم بناؤه في Dockerfile غنيمة). +* الصورة الصغيرة يسهل سحبها من **Ghanimah Container Registry** عبر شبكة غنيمة السريعة خلال مسبارات زمنية تقل عن **200ms**. + +### 2. التوسع الاستباقي (Proactive Pre-Warming & Buffer Capacity) +* ضبط الحد الأدنى للحاويات عند 39 حاوية، وتفعيل التوسع التلقائي (HPA) فور وصول استهلاك الذاكرة أو المعالج إلى **70%** (بدلاً من 90%). +* هذا يمنح الـ Clusters وقتاً كافياً لإطلاق الحاويات قبل وصول الضغط الفعلي للذروة. + +### 3. تحسين فحوصات الجاهزية (Readiness Probes Tuning) +* ضبط فحوصات الجاهزية للبدء سريعاً بدون تأخير غير برمجيات: + ```yaml + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 1 # البدء بالفحص فوراً بعد ثانية واحدة + periodSeconds: 2 # الفحص كل ثانيتين + successThreshold: 1 + failureThreshold: 2 + ``` + +### 4. تسريع وقت تشغيل التطبيق (Fast Application Startup) +* الاعتماد على لغة Go المجمعة بلغة الآلة (Native Compiled Output) والتي تبدأ العمل خلال أجزاء من الملي ثانية بدون تجمعات JVM أو حزم تفسير ثقيلة. +* تجميع وإيقاف أي اتصالات قاعدة بيانات كسولة (Lazy Initialization) واستبدالها باتصالات جاهزة سلفاً (Pre-warmed connection pools). + +--- + +## 4️⃣ استخدام Ghanimah Block Storage للبيانات المستمرة (Stateful Workloads) + +تعتمد الحاويات بطبيعتها على كونها **Stateless** (تزول بياناتها بزوال الحاوية). ولتشغيل التطبيقات التي تتطلب حفظ البيانات بشكل دائم (Stateful Workloads مثل قواعد البيانات PostgreSQL و Redis Persistent Logs)، توفر منصة غنيمة **Ghanimah Block Storage (GBS)**. + +### 🔑 أهم الميزات والية العمل: + +```text +┌────────────────────────┐ Persistent Volume Claim ┌────────────────────────────┐ +│ PostgreSQL Pod │ ───────────────────────────────────► │ Ghanimah Block Storage(PV) │ +│ (Stateful Workload) │ (Attach NVMe Storage) │ (High-Performance SSD) │ +└────────────────────────┘ └────────────────────────────┘ +``` + +1. **الأداء العالي (High Performance NVMe Volumes):** + * يوفر Ghanimah Block Storage أقراص NVMe فائقة السرعة مع معدل عمليات إدخال/إخراج يصل إلى **60,000 IOPS** وتأخير أقل من **1ms**، وهو مثالي لقواعد البيانات الضخمة. + +2. **التكامل عبر Kubernetes Dynamic Provisioning (PVC & StorageClass):** + * يتم ربط التخزين بالتطبيقات باستخدام `PersistentVolumeClaim` (PVC) وتحديد `StorageClass: ghanimah-block-nvme`: + ```yaml + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ghanimah-db-pvc + spec: + accessModes: + - ReadWriteOnce + storageClassName: ghanimah-block-nvme + resources: + requests: + storage: 250Gi + ``` + +3. **الحماية والاستمرارية (Data Persistence & High Availability):** + * عند تعطل الـ Pod المربوط بوحدة التخزين، تقوم منصة غنيمة تلقائياً بفصل قرص الـ Block Storage وإعادة ربطه (`Attach/Detach`) بالـ Pod الجديد عبر عقدة أخرى بدون أي فقدان للبيانات. + +4. **النسخ الاحتياطي واللقطات الفورية (Snapshots & Replication):** + * يدعم Ghanimah Block Storage إنشاء لقطات فورية (Volume Snapshots) دورية بدون التأثير على أداء الخدمة الحية، مع إمكانية استرجاعها فوراً في حالات الطوارئ (Disaster Recovery). diff --git a/q5-mithal-monitor/dashbourd.html b/q5-mithal-monitor/dashbourd.html new file mode 100644 index 0000000..301f52d --- /dev/null +++ b/q5-mithal-monitor/dashbourd.html @@ -0,0 +1,337 @@ + + + + + + + + Mithal Monitoring Dashboard + + + + + + + + + +

📈 Mithal Monitoring Dashboard

+ +
+ +
+ +

Uptime (24h)

+ +
--
+ +
+ +
+ +

Average Latency

+ +
--
+ +
+ +
+ +

SSL Days Left

+ +
--
+ +
+ +
+ +

Total Checks

+ +
--
+ +
+ +
+ +
+ + + +
+ +

Last 10 Checks

+ + + + + + + + + + + + + + + + + + + + + + +
TimeStatusLatencyDNSSSLSearch
+ +
+ + Refreshes every 60 seconds + +
+ + + + + + \ No newline at end of file diff --git a/q5-mithal-monitor/metrics.json b/q5-mithal-monitor/metrics.json new file mode 100644 index 0000000..a75f364 --- /dev/null +++ b/q5-mithal-monitor/metrics.json @@ -0,0 +1,107 @@ +[ + { + "timestamp": "2026-07-28T09:59:01.075371", + "latency_ms": 1021.86, + "status_code": 200, + "uptime": true, + "dns_lookup_ms": 6.92, + "ssl": { + "expires_at": "2026-09-15T13:10:47", + "days_left": 49 + }, + "search": { + "status": 200, + "response_ms": 958.41 + } + }, + { + "timestamp": "2026-07-28T10:00:03.486672", + "latency_ms": 1078.94, + "status_code": 200, + "uptime": true, + "dns_lookup_ms": 7.81, + "ssl": { + "expires_at": "2026-09-15T13:10:47", + "days_left": 49 + }, + "search": { + "status": 200, + "response_ms": 875.35 + } + }, + { + "timestamp": "2026-07-28T10:01:05.991518", + "latency_ms": 1063.59, + "status_code": 200, + "uptime": true, + "dns_lookup_ms": 6.1, + "ssl": { + "expires_at": "2026-09-15T13:10:47", + "days_left": 49 + }, + "search": { + "status": 200, + "response_ms": 872.83 + } + }, + { + "timestamp": "2026-07-28T10:02:08.495565", + "latency_ms": 1022.08, + "status_code": 200, + "uptime": true, + "dns_lookup_ms": 5.8, + "ssl": { + "expires_at": "2026-09-15T13:10:47", + "days_left": 49 + }, + "search": { + "status": 200, + "response_ms": 919.46 + } + }, + { + "timestamp": "2026-07-28T10:03:10.815308", + "latency_ms": 944.67, + "status_code": 200, + "uptime": true, + "dns_lookup_ms": 7.7, + "ssl": { + "expires_at": "2026-09-15T13:10:47", + "days_left": 49 + }, + "search": { + "status": 200, + "response_ms": 883.18 + } + }, + { + "timestamp": "2026-07-28T10:04:12.942249", + "latency_ms": 809.59, + "status_code": 200, + "uptime": true, + "dns_lookup_ms": 5.28, + "ssl": { + "expires_at": "2026-09-15T13:10:47", + "days_left": 49 + }, + "search": { + "status": 200, + "response_ms": 849.83 + } + }, + { + "timestamp": "2026-07-28T10:05:15.376909", + "latency_ms": 909.74, + "status_code": 200, + "uptime": true, + "dns_lookup_ms": 4.95, + "ssl": { + "expires_at": "2026-09-15T13:10:47", + "days_left": 49 + }, + "search": { + "status": 200, + "response_ms": 803.56 + } + } +] \ No newline at end of file diff --git a/q5-mithal-monitor/monitor.py b/q5-mithal-monitor/monitor.py new file mode 100644 index 0000000..57e5cab --- /dev/null +++ b/q5-mithal-monitor/monitor.py @@ -0,0 +1,204 @@ +import requests +import socket +import ssl +import json +import os +import time +from datetime import datetime + +# ----------------------------- +# Configuration +# ----------------------------- +BASE_URL = "https://mithal.space" +SEARCH_URL = "https://mithal.space/search?q=test" + +OUTPUT_FILE = "metrics.json" +INTERVAL = 60 # seconds + +MAX_RECORDS = 1440 # 24 hours (1 record/min) + + +# ----------------------------- +# HTTP Latency + Uptime +# ----------------------------- +def check_http(): + start = time.perf_counter() + + try: + response = requests.get(BASE_URL, timeout=10) + + latency = round((time.perf_counter() - start) * 1000, 2) + + return { + "status_code": response.status_code, + "uptime": response.status_code == 200, + "latency_ms": latency + } + + except Exception: + latency = round((time.perf_counter() - start) * 1000, 2) + + return { + "status_code": None, + "uptime": False, + "latency_ms": latency + } + + +# ----------------------------- +# DNS Lookup Time +# ----------------------------- +def check_dns(): + host = BASE_URL.replace("https://", "").replace("http://", "").split("/")[0] + + start = time.perf_counter() + + try: + socket.gethostbyname(host) + dns_time = round((time.perf_counter() - start) * 1000, 2) + + except Exception: + dns_time = None + + return dns_time + + +# ----------------------------- +# SSL Expiry +# ----------------------------- +def check_ssl(): + host = BASE_URL.replace("https://", "").replace("http://", "").split("/")[0] + + try: + context = ssl.create_default_context() + + with context.wrap_socket( + socket.socket(socket.AF_INET), + server_hostname=host + ) as s: + + s.settimeout(10) + s.connect((host, 443)) + + cert = s.getpeercert() + + expire = datetime.strptime( + cert["notAfter"], + "%b %d %H:%M:%S %Y %Z" + ) + + remaining = (expire - datetime.utcnow()).days + + return { + "expires_at": expire.isoformat(), + "days_left": remaining + } + + except Exception: + + return { + "expires_at": None, + "days_left": None + } + + +# ----------------------------- +# Search Response Time +# ----------------------------- +def check_search(): + start = time.perf_counter() + + try: + response = requests.get( + SEARCH_URL, + timeout=10 + ) + + elapsed = round((time.perf_counter() - start) * 1000, 2) + + return { + "status": response.status_code, + "response_ms": elapsed + } + + except Exception: + + elapsed = round((time.perf_counter() - start) * 1000, 2) + + return { + "status": None, + "response_ms": elapsed + } + + +# ----------------------------- +# Load JSON +# ----------------------------- +def load_metrics(): + + if not os.path.exists(OUTPUT_FILE): + return [] + + try: + with open(OUTPUT_FILE, "r") as f: + return json.load(f) + + except Exception: + return [] + + +# ----------------------------- +# Save JSON +# ----------------------------- +def save_metrics(data): + + with open(OUTPUT_FILE, "w") as f: + json.dump(data, f, indent=4) + + +# ----------------------------- +# Main Monitor Loop +# ----------------------------- +def monitor(): + + print("Starting Mithal Monitor...") + + while True: + + http = check_http() + dns = check_dns() + ssl_info = check_ssl() + search = check_search() + + record = { + "timestamp": datetime.utcnow().isoformat(), + + "latency_ms": http["latency_ms"], + + "status_code": http["status_code"], + + "uptime": http["uptime"], + + "dns_lookup_ms": dns, + + "ssl": ssl_info, + + "search": search + } + + data = load_metrics() + + data.append(record) + + if len(data) > MAX_RECORDS: + data = data[-MAX_RECORDS:] + + save_metrics(data) + + print(record) + + time.sleep(INTERVAL) + + +if __name__ == "__main__": + monitor() \ No newline at end of file