diff --git a/common-qabilah/qabilah-profile.txt b/common-qabilah/qabilah-profile.txt new file mode 100644 index 0000000..a4f8e94 --- /dev/null +++ b/common-qabilah/qabilah-profile.txt @@ -0,0 +1,21 @@ +Name: Mohamed Wael + +Track: SRE Internship + +GitHub Repository: +ghaymah-exam-mohamed-wael-sre + +Qabilah Profile: +https://qabilah.com/profile/moxwael + +Skills: +- Linux +- Docker +- Kubernetes +- AWS +- Terraform +- GitHub Actions +- Prometheus +- Grafana +- Python +- Monitoring diff --git a/q1-deploy-monitor/Dashboard b/q1-deploy-monitor/Dashboard new file mode 100644 index 0000000..7e87ead --- /dev/null +++ b/q1-deploy-monitor/Dashboard @@ -0,0 +1,15 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +RUN chmod +x start.sh + +EXPOSE 5000 + +CMD ["./start.sh"] diff --git a/q1-deploy-monitor/dashboard.html b/q1-deploy-monitor/dashboard.html new file mode 100644 index 0000000..76eb35c --- /dev/null +++ b/q1-deploy-monitor/dashboard.html @@ -0,0 +1,111 @@ + + + + + +SRE Dashboard + + + + + + + +

API Monitoring Dashboard

+ + + + + + + + + + + + + + + + + + + +
TimestampStatusResponse Time
+ + + + + diff --git a/q1-deploy-monitor/health-check.sh b/q1-deploy-monitor/health-check.sh new file mode 100644 index 0000000..87a0981 --- /dev/null +++ b/q1-deploy-monitor/health-check.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +URL="http://localhost:5000/health" + +STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$URL") + +if [ "$STATUS" -eq 200 ]; then + echo "Healthy" + exit 0 +else + echo "Unhealthy" + exit 1 +fi diff --git a/q2-postmortem/postmortem-report.md b/q2-postmortem/postmortem-report.md new file mode 100644 index 0000000..ec285a1 --- /dev/null +++ b/q2-postmortem/postmortem-report.md @@ -0,0 +1,123 @@ +# Incident Postmortem Report + +## Incident Summary + +On 27 July 2026, an application deployed on the Ghaymah platform experienced repeated **OOMKilled** events, resulting in approximately **45 minutes of downtime**. During this period, users were unable to access the service because the application pods kept restarting after exceeding their memory limit. + +--- + +# Timeline + +| Time | Event | +|------|-------| +| 10:00 | A new application version was deployed. | +| 10:08 | Memory usage began increasing steadily. | +| 10:12 | The first pod was terminated with an **OOMKilled** event. | +| 10:15 | Kubernetes automatically restarted the pod. | +| 10:18 – 10:40 | The application entered a restart loop as multiple pods were repeatedly OOMKilled. | +| 10:42 | The SRE team investigated pod events, logs, and resource metrics. | +| 10:45 | Memory limits were increased temporarily, and the application's memory usage was reviewed. | +| 10:47 | A new healthy pod started successfully. | +| 10:55 | The service was fully restored and became available to users again. | + +--- + +# Root Cause + +The application gradually consumed more memory than the Kubernetes memory limit that had been configured for the container. Once the limit was exceeded, Kubernetes terminated the container with an **OOMKilled** event. + +Although Kubernetes restarted the pod automatically, the application continued to exceed its memory limit after each restart. This created a restart loop that kept the service unavailable until the memory issue was addressed. + +--- + +# Impact + +- Approximately **45 minutes** of service downtime. +- Multiple pod restarts due to repeated **OOMKilled** events. +- Users experienced failed requests and service unavailability. +- Increased operational effort to investigate and restore the application. + +--- + +# Resolution + +The incident was resolved by: + +- Investigating Kubernetes pod events and application logs. +- Identifying excessive memory consumption. +- Increasing the container memory limit temporarily. +- Restarting the affected pods. +- Verifying that the application returned to a healthy state. + +--- + +# Recommendations + +To reduce the likelihood of similar incidents: + +- Configure appropriate CPU and memory requests and limits. +- Enable Horizontal Pod Autoscaler (HPA). +- Create alerts for high memory utilization. +- Monitor pod restart counts and OOMKilled events. +- Perform load testing before deploying new releases. +- Review application memory usage regularly to identify potential memory leaks. + +--- + +# Auto-Scaling Policy for Ghaymah Platform + +To prevent similar incidents, the platform should automatically scale the application based on resource utilization. + +## Horizontal Pod Autoscaler (HPA) + +- Minimum replicas: **2** +- Maximum replicas: **10** + +### Scaling Rules + +- Scale out when CPU utilization exceeds **70%**. +- Scale out when memory utilization exceeds **75%**. +- Scale in only after resource utilization remains below **40%** for several minutes to avoid frequent scaling. + +This policy helps distribute incoming traffic across multiple pods before any single container reaches its memory limit. + +--- + +# Early Detection Using Ghaymah Monitoring + +The issue can be detected before it causes downtime by continuously monitoring application and Kubernetes metrics. + +## Key Metrics + +- Container memory usage +- Memory utilization percentage +- CPU utilization +- Pod restart count +- OOMKilled events +- API response time +- HTTP 5xx error rate +- Application availability + +## Alerting Rules + +Create alerts when: + +- Memory utilization exceeds **80%** for more than 5 minutes. +- A pod restarts more than **3 times** within 10 minutes. +- An **OOMKilled** event occurs. +- API response time exceeds the defined threshold. +- Service availability drops below the expected level. + +## Dashboard + +A monitoring dashboard should display: + +- CPU usage +- Memory usage +- Running pods +- Pod restart count +- Request rate +- Response time +- Active alerts + +With these metrics and alerts in place, the operations team can detect abnormal memory growth early and take corrective action before users are affected. diff --git a/q3-cicd/workflow.yml b/q3-cicd/workflow.yml new file mode 100644 index 0000000..c8eb2b3 --- /dev/null +++ b/q3-cicd/workflow.yml @@ -0,0 +1,86 @@ +name: Build & Deploy to Ghaymah + +on: + push: + branches: + - main + +jobs: + build-and-push: + name: Build and Push Docker Image + runs-on: ubuntu-latest + + steps: + - name: Checkout Source Code + uses: actions/checkout@v4 + + - name: Build Docker Image + run: | + docker build -t ghaymah-app:${{ github.sha }} . + + # Login to Ghaymah Container Registry + # Store credentials as GitHub Secrets + - name: Login to Ghaymah Registry + run: | + echo "${{ secrets.GHAYMAH_REGISTRY_PASSWORD }}" | docker login ${{ secrets.GHAYMAH_REGISTRY }} \ + -u "${{ secrets.GHAYMAH_REGISTRY_USERNAME }}" \ + --password-stdin + + - name: Tag Docker Image + run: | + docker tag ghaymah-app:${{ github.sha }} \ + ${{ secrets.GHAYMAH_REGISTRY }}/ghaymah-app:${{ github.sha }} + + - name: Push Docker Image + run: | + docker push \ + ${{ secrets.GHAYMAH_REGISTRY }}/ghaymah-app:${{ github.sha }} + + deploy-staging: + name: Deploy to Staging + needs: build-and-push + runs-on: ubuntu-latest + environment: staging + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Install Ghaymah CLI + run: | + curl -sSl https://cli.ghaymah.systems/install.sh | bash + + - name: Login to Ghaymah + run: | + gy auth login --token "${{ secrets.GHAYMAH_API_TOKEN }}" + + - name: Deploy to Staging + run: | + gy resource app launch + + deploy-production: + name: Deploy to Production + needs: deploy-staging + runs-on: ubuntu-latest + + # Configure this environment in GitHub: + # Settings → Environments → production + # Add Required Reviewers to enable manual approval. + environment: + name: production + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Install Ghaymah CLI + run: | + curl -sSl https://cli.ghaymah.systems/install.sh | bash + + - name: Login to Ghaymah + run: | + gy auth login --token "${{ secrets.GHAYMAH_API_TOKEN }}" + + - name: Deploy to Production + run: | + gy resource app launch diff --git a/q4-scalability/architecture.png b/q4-scalability/architecture.png new file mode 100644 index 0000000..9318124 Binary files /dev/null and b/q4-scalability/architecture.png differ diff --git a/q4-scalability/calculations.md b/q4-scalability/calculations.md new file mode 100644 index 0000000..94766df --- /dev/null +++ b/q4-scalability/calculations.md @@ -0,0 +1,66 @@ +# Scalability Design + +## 1. Architecture + +The application is deployed on a Kubernetes cluster hosted on Ghaymah Cloud. + +Incoming traffic is distributed through a Ghaymah Load Balancer across multiple application containers. The application stores persistent data using Ghaymah Block Storage, which is mounted as a Persistent Volume. + +--- + +## 2. Required Number of Containers + +### Given + +- Incoming traffic = **15,000 requests/second** +- One container handles = **500 requests/second** +- Safety margin = **30%** + +### Step 1 + +Base number of containers + +15000 / 500 = 30 containers + +### Step 2 + +Add 30% capacity + +30 × 1.3 = 39 containers + +### Final Answer + +**39 containers** + +--- + +## 3. Cold Start Strategy + +To reduce startup latency for newly created containers: + +- Keep a minimum number of warm replicas running. +- Use Kubernetes readiness probes before routing traffic. +- Pull container images in advance on cluster nodes. +- Use Horizontal Pod Autoscaler (HPA) to scale before traffic reaches peak levels. +- Deploy applications with rolling updates to avoid downtime. + +--- + +## 4. Using Ghaymah Block Storage + +Ghaymah Block Storage provides persistent storage for stateful workloads. + +Typical use cases include: + +- Databases +- PostgreSQL +- MySQL +- Redis persistence +- Application uploaded files +- Persistent logs + +Benefits: + +- Data persists even if a container is recreated. +- Storage can be attached to new containers. +- Reliable and durable storage for production workloads. diff --git a/q5-mithal-monitor/dashboard.html b/q5-mithal-monitor/dashboard.html new file mode 100644 index 0000000..a05e8df --- /dev/null +++ b/q5-mithal-monitor/dashboard.html @@ -0,0 +1,292 @@ + + + + + +Mithal Monitoring Dashboard + + + + + + + + + +

Mithal Monitoring Dashboard

+ +
+ +
+ +

Uptime (24h)

+ +

--

+ +
+ +
+ +

SSL Remaining

+ +

--

+ +
+ +
+ +

Latest Latency

+ +

--

+ +
+ +
+ +
+ + + +
+ +

Last 10 Checks

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
TimeStatusLatencyDNSSearchSSL Days
+ + + + + + diff --git a/q5-mithal-monitor/monitor.py b/q5-mithal-monitor/monitor.py new file mode 100644 index 0000000..0bb0175 --- /dev/null +++ b/q5-mithal-monitor/monitor.py @@ -0,0 +1,177 @@ +import requests +import socket +import ssl +import json +import time +import threading +from datetime import datetime +from flask import Flask, jsonify, send_from_directory + +app = Flask(__name__) + +BASE_URL = "https://mithal.space" +SEARCH_URL = "https://mithal.space/search?q=test" + +DATA_FILE = "monitor_data.json" + +CHECK_INTERVAL = 60 + + +def load_data(): + try: + with open(DATA_FILE, "r") as f: + return json.load(f) + except: + return [] + + +def save_data(data): + with open(DATA_FILE, "w") as f: + json.dump(data, f, indent=4) + + +def get_latency(): + start = time.time() + + response = requests.get(BASE_URL, timeout=10) + + latency = round((time.time() - start) * 1000, 2) + + return latency, response.status_code + + +def get_dns_time(): + + start = time.time() + + socket.gethostbyname("mithal.space") + + return round((time.time() - start) * 1000, 2) + + +def get_ssl_info(): + + hostname = "mithal.space" + + context = ssl.create_default_context() + + with context.wrap_socket( + socket.socket(), + server_hostname=hostname + ) as s: + + s.settimeout(10) + + s.connect((hostname, 443)) + + cert = s.getpeercert() + + expiry = datetime.strptime( + cert["notAfter"], + "%b %d %H:%M:%S %Y %Z" + ) + + remaining = (expiry - datetime.utcnow()).days + + return expiry.strftime("%Y-%m-%d"), remaining + + +def get_search_latency(): + + start = time.time() + + response = requests.get( + SEARCH_URL, + timeout=10 + ) + + latency = round((time.time() - start) * 1000, 2) + + return latency, response.status_code + +def monitor(): + + while True: + + try: + + latency, status = get_latency() + + dns = get_dns_time() + + ssl_expiry, ssl_days = get_ssl_info() + + search_latency, search_status = get_search_latency() + + entry = { + + "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + + "uptime": status == 200, + + "status_code": status, + + "latency_ms": latency, + + "dns_ms": dns, + + "ssl_expiry": ssl_expiry, + + "ssl_remaining_days": ssl_days, + + "search_latency_ms": search_latency, + + "search_status": search_status + + } + + data = load_data() + + data.append(entry) + + data = data[-1440:] + + save_data(data) + + print(entry) + + except Exception as e: + + print("Monitoring Error:", e) + + time.sleep(CHECK_INTERVAL) + + +@app.route("/api/data") +def api_data(): + + return jsonify(load_data()) + + +@app.route("/") +def home(): + + return { + "message": "Mithal Monitoring API", + "dashboard": "/dashboard" + } + + +@app.route("/dashboard") +def dashboard(): + + return send_from_directory(".", "dashboard.html") + + +if __name__ == "__main__": + + thread = threading.Thread(target=monitor) + + thread.daemon = True + + thread.start() + + app.run( + host="0.0.0.0", + port=5002 + ) diff --git a/q5-mithal-monitor/monitor_data.json b/q5-mithal-monitor/monitor_data.json new file mode 100644 index 0000000..adaa94f --- /dev/null +++ b/q5-mithal-monitor/monitor_data.json @@ -0,0 +1,13 @@ +[ + { + "timestamp": "2026-07-28 15:09:54", + "uptime": true, + "status_code": 200, + "latency_ms": 1388.97, + "dns_ms": 65.85, + "ssl_expiry": "2026-09-15", + "ssl_remaining_days": 49, + "search_latency_ms": 1204.14, + "search_status": 200 + } +] \ No newline at end of file