commit 1150963ef4a0cd6cc217cccb3f61e237caed7b77 Author: mostafa Date: Tue Jul 28 13:34:58 2026 +0300 intial commit diff --git a/common-qabilah/qabilah-profile.md b/common-qabilah/qabilah-profile.md new file mode 100644 index 0000000..dd4d1ec --- /dev/null +++ b/common-qabilah/qabilah-profile.md @@ -0,0 +1 @@ +https://qabilah.com/profile/mostafa-bebars/posts \ No newline at end of file diff --git a/q1-deploy-monitor/Dockerfile b/q1-deploy-monitor/Dockerfile new file mode 100644 index 0000000..de3e077 --- /dev/null +++ b/q1-deploy-monitor/Dockerfile @@ -0,0 +1,18 @@ +# Use a lightweight Node base image +FROM node:18-alpine + +# Set the working directory inside the container +WORKDIR /usr/src/app + +# Copy dependency definitions and install them +COPY package*.json ./ +RUN npm install --production + +# Copy the application code +COPY . . + +# Expose the API port +EXPOSE 3000 + +# Start the application +CMD ["npm", "start"] \ No newline at end of file diff --git a/q1-deploy-monitor/dashboard.html b/q1-deploy-monitor/dashboard.html new file mode 100644 index 0000000..6526e34 --- /dev/null +++ b/q1-deploy-monitor/dashboard.html @@ -0,0 +1,105 @@ + + + + + + API Metrics Dashboard + + + +

System Status

Avg Response Time

0 ms

Dashboard Pings

+ + + + \ No newline at end of file diff --git a/q1-deploy-monitor/monitor.sh b/q1-deploy-monitor/monitor.sh new file mode 100644 index 0000000..4f8d711 --- /dev/null +++ b/q1-deploy-monitor/monitor.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Replace this with your actual deployed Ghaymah URL +API_URL="https://bebars-64f19cdda81b.hosted.ghaymah.systems/health" + +echo "Starting monitoring for $API_URL..." +echo "Press [CTRL+C] to stop." + +while true; do + TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S") + + # Fetch only the HTTP status code + HTTP_STATUS=$(curl -o /dev/null -s -w "%{http_code}" "$API_URL") + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[$TIMESTAMP] ✅ Status: $HTTP_STATUS - App is HEALTHY" + else + echo "[$TIMESTAMP] ❌ Status: $HTTP_STATUS - App is DOWN or UNREACHABLE" + fi + + sleep 30 +done \ No newline at end of file diff --git a/q1-deploy-monitor/package.json b/q1-deploy-monitor/package.json new file mode 100644 index 0000000..0ca8fb5 --- /dev/null +++ b/q1-deploy-monitor/package.json @@ -0,0 +1,18 @@ +{ + "name": "q1-deploy-monitor", + "version": "1.0.0", + "description": "", + "main": "server.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node server.js" + }, + "keywords": [], + "author": "", + "license": "ISC", + "type": "commonjs", + "dependencies": { + "cors": "^2.8.5", + "express": "^4.18.2" + } +} diff --git a/q1-deploy-monitor/server.js b/q1-deploy-monitor/server.js new file mode 100644 index 0000000..404221e --- /dev/null +++ b/q1-deploy-monitor/server.js @@ -0,0 +1,23 @@ +const express = require('express'); +const cors = require('cors'); + +const app = express(); +const port = process.env.PORT || 3000; + +let requestCount = 0; +let totalResponseTime = 0; + +// cors +app.use(cors()); + + + +// The requested health endpoint +app.get('/health', (req, res) => { + res.status(200).json({ status: 'UP', timestamp: new Date().toISOString() }); +}); + + +app.listen(port, () => { + console.log(`API running on port ${port}`); +}); \ No newline at end of file diff --git a/q2-postmortem/postmortem-report.md b/q2-postmortem/postmortem-report.md new file mode 100644 index 0000000..b16ca8d --- /dev/null +++ b/q2-postmortem/postmortem-report.md @@ -0,0 +1,51 @@ +# Incident Postmortem Analysis: Application Outage (OOMKilled) + +## 1. Summary +The application deployed on the Ghaymah platform experienced a complete outage lasting 45 minutes. The service continuously crashed and failed to start due to recurring `OOMKilled` (Out of Memory) errors. This resulted in a 100% failure rate for incoming requests during the incident window until mitigation measures were applied. + +## 2. Timeline of Events + +* **T-00:00 - Traffic Spike / Memory Leak Initiated:** Application memory consumption begins to spike uncontrollably due to an anomalous load or internal memory leak. +* **T+00:05 - Initial OOMKilled Event:** Container RAM usage hits the hard limit. The Ghaymah orchestrator forcefully terminates the process to protect the node, logging an `OOMKilled` event. +* **T+00:06 to T+00:44 - CrashLoopBackOff State:** Ghaymah automatically attempts to restart the container, but it repeatedly exceeds memory limits shortly after booting. Service remains entirely unavailable. +* **T+00:45 - Incident Mitigated:** Engineers manually intervene by vertically scaling the container's memory limits or deploying a hotfix. The application stabilizes and resumes serving traffic. + +## 3. Root Cause +An `OOMKilled` error occurs when a container attempts to consume more memory than its configured limits allow. The underlying causes typically involve: +* **Memory Leak:** A flaw in the application code where unused memory is not released (e.g., inefficient garbage collection, unclosed connections). +* **Unoptimized Queries/Payloads:** The application attempted to process an unusually large dataset in memory all at once rather than using streams or pagination. +* **Under-provisioning:** The application lacked the baseline memory required to handle a legitimate surge in concurrent user traffic. + +## 4. Action Items & Recommendations +* **Code Profiling:** Conduct memory profiling on the application to identify and patch memory leaks. +* **Pagination/Streaming:** Ensure large data processing tasks use streams rather than loading entire datasets into RAM. +* **Capacity Tuning:** Re-evaluate and adjust the baseline memory limits and requests configured for the container. +* **Implement Auto-scaling:** Configure dynamic scaling policies to absorb sudden spikes automatically. + +--- + +## Auto-Scaling Policy Design for Ghaymah + +Ghaymah natively supports both horizontal and vertical auto-scaling based on CPU and memory usage. To prevent this issue from recurring, a proactive scaling policy should be implemented: + +* **Target Metric:** Average Container Memory Utilization. +* **Scale-Out (Increase Capacity):** + * **Condition:** If Memory Utilization is greater than 75% for a sustained period of 2 minutes. + * **Action:** Add 1 additional container replica (Horizontal Scaling) or dynamically increase the RAM allocation (Vertical Scaling). +* **Scale-In (Decrease Capacity):** + * **Condition:** If Memory Utilization falls below 40% for 5 minutes. + * **Action:** Remove 1 replica to optimize resource consumption. +* **Limits:** Minimum of 2 replicas (for high availability) and a Maximum of 10 replicas (to control cloud spend). + +**Why this works:** The 75% threshold leaves a 25% safety buffer. This gives the Ghaymah platform enough time to spin up new instances and distribute the load before any single container hits the 100% `OOMKilled` threshold. + +--- + +## Early Detection Using Ghaymah Monitoring Tools + +Relying on user complaints or complete downtime is an anti-pattern. You can detect memory exhaustion early using Ghaymah's built-in monitoring: + +* **Threshold Alerts:** Configure Ghaymah to send automated alerts (via Slack, email, or webhook) when memory usage reaches 70% and 80%. This provides a critical window for intervention before the container crashes. +* **Restart Rate Monitoring:** Set an alert if the container restart count exceeds 1 within a 15-minute window. Frequent restarts are the earliest indicator of a `CrashLoopBackOff` state. +* **Application Logs Analysis:** Monitor logs for warning signs such as garbage collection (GC) taking excessively long, or application-level out-of-memory warnings that often precede the infrastructure-level kill signal. +* **Endpoint Health Checks:** Ensure proper Liveness and Readiness probes are configured. If high memory pressure degrades application performance, the Readiness probe should fail, instructing Ghaymah's load balancer to stop routing traffic to the struggling instance before it dies. diff --git a/q3-cicd/.ghaymah.json b/q3-cicd/.ghaymah.json new file mode 100644 index 0000000..ac07ee9 --- /dev/null +++ b/q3-cicd/.ghaymah.json @@ -0,0 +1,17 @@ +{ + "id": "9299b6b6-ff2e-4984-8d85-5bdb0b809533", + "name": "q3", + "projectId": "27aacb50-4d56-474c-baae-b52a853b4d57", + "ports": [ + { + "expose": true, + "number": 3000 + } + ], + "publicAccess": { + "enabled": true, + "domain": "auto" + }, + "resourceTier": "t1", + "dockerFileName": "Dockerfile" +} diff --git a/q3-cicd/.github/workflows/ci.yml b/q3-cicd/.github/workflows/ci.yml new file mode 100644 index 0000000..5feaf17 --- /dev/null +++ b/q3-cicd/.github/workflows/ci.yml @@ -0,0 +1,55 @@ + +name: Ghaymah CI/CD Pipeline + +on: + push: + branches: [ "master" ] + + + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + deploy-staging: + name: Deploy to Staging + needs: build + runs-on: ubuntu-latest + # Staging is used for pre-production validation before a live release. + environment: + name: staging + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Install Ghaymah CLI + run: curl -sSL https://cli.ghaymah.systems/install.sh | bash + + - name: Login to Ghaymah + run: $HOME/ghaymah/bin/gy auth login --email "${{ secrets.GHAYMAH_EMAIL }}" --password "${{ secrets.GHAYMAH_PW }}" --debug + + + + deploy-production: + name: Deploy to Production + needs: deploy-staging + runs-on: ubuntu-latest + # Production is the live environment and should be protected by manual approval. + environment: + name: production + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Install Ghaymah CLI + run: curl -sSL https://cli.ghaymah.systems/install.sh | bash + + - name: Login to Ghaymah + run: $HOME/ghaymah/bin/gy auth login --email "${{secrets.GHAYMAH_EMAIL}}" --password "${{secrets.GHAYMAH_PW}}" --debug + + - name: Deploy to Ghaymah (Production) + run: $HOME/ghaymah/bin/gy resource app launch --debug diff --git a/q3-cicd/Dockerfile b/q3-cicd/Dockerfile new file mode 100644 index 0000000..de3e077 --- /dev/null +++ b/q3-cicd/Dockerfile @@ -0,0 +1,18 @@ +# Use a lightweight Node base image +FROM node:18-alpine + +# Set the working directory inside the container +WORKDIR /usr/src/app + +# Copy dependency definitions and install them +COPY package*.json ./ +RUN npm install --production + +# Copy the application code +COPY . . + +# Expose the API port +EXPOSE 3000 + +# Start the application +CMD ["npm", "start"] \ No newline at end of file diff --git a/q3-cicd/README.md b/q3-cicd/README.md new file mode 100644 index 0000000..029d056 --- /dev/null +++ b/q3-cicd/README.md @@ -0,0 +1,50 @@ +# Ghaymah CI/CD Guide + +## Staging vs. Production + +- Staging is a pre-production environment used to test changes safely before they reach real users. It is typically isolated from production data and traffic, so teams can validate deployments, configuration, and application behavior without affecting live services. +- Production is the live environment that serves real users and business-critical traffic. Deployments here should be controlled carefully, usually with approval gates, protected environments, and rollback plans. +- A common release flow is: + 1. Build and test the application. + 2. Deploy to staging. + 3. Validate the release. + 4. Request approval for production. + 5. Deploy to production. + +## Connecting the GitHub Actions workflow to Ghaymah CLI + +The workflow uses the Ghaymah CLI to authenticate and deploy the app. + +### 1. Add GitHub secrets +Create these repository or environment secrets in GitHub: + +- GHAYMAH_EMAIL +- GHAYMAH_PW + +### 2. Install the CLI +The workflow installs the CLI during the deployment job: + +```bash +curl -sSL https://cli.ghaymah.systems/install.sh | bash +``` + +### 3. Authenticate with Ghaymah +After installation, the workflow logs in with the stored credentials: + +```bash +$HOME/ghaymah/bin/gy auth login --email "${{ secrets.GHAYMAH_EMAIL }}" --password "${{ secrets.GHAYMAH_PW }}" --debug +``` + +### 4. Deploy the application +The deployment step then launches the app through Ghaymah: + +```bash +$HOME/ghaymah/bin/gy resource app launch --debug +``` + +### 5. Recommended GitHub environment setup +Use GitHub Environments for deployment control: + +- Create a staging environment for regular validation. +- Create a production environment and require manual approval before deployment. +- Attach the same secrets to the relevant environment when needed. diff --git a/q3-cicd/package.json b/q3-cicd/package.json new file mode 100644 index 0000000..0ca8fb5 --- /dev/null +++ b/q3-cicd/package.json @@ -0,0 +1,18 @@ +{ + "name": "q1-deploy-monitor", + "version": "1.0.0", + "description": "", + "main": "server.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node server.js" + }, + "keywords": [], + "author": "", + "license": "ISC", + "type": "commonjs", + "dependencies": { + "cors": "^2.8.5", + "express": "^4.18.2" + } +} diff --git a/q3-cicd/server.js b/q3-cicd/server.js new file mode 100644 index 0000000..404221e --- /dev/null +++ b/q3-cicd/server.js @@ -0,0 +1,23 @@ +const express = require('express'); +const cors = require('cors'); + +const app = express(); +const port = process.env.PORT || 3000; + +let requestCount = 0; +let totalResponseTime = 0; + +// cors +app.use(cors()); + + + +// The requested health endpoint +app.get('/health', (req, res) => { + res.status(200).json({ status: 'UP', timestamp: new Date().toISOString() }); +}); + + +app.listen(port, () => { + console.log(`API running on port ${port}`); +}); \ No newline at end of file diff --git a/q4-scalability/architecture.png b/q4-scalability/architecture.png new file mode 100644 index 0000000..f033c4a Binary files /dev/null and b/q4-scalability/architecture.png differ diff --git a/q4-scalability/calculations.md b/q4-scalability/calculations.md new file mode 100644 index 0000000..b927c89 --- /dev/null +++ b/q4-scalability/calculations.md @@ -0,0 +1,20 @@ +Container Calculation + + +### Container Capacity Calculation +To handle 15,000 requests per second with a base container capacity of 500 requests per second and a 30% safety margin: + +1. **Base Containers Needed:** 15,000 / 500 = 30 containers +2. **Safety Buffer (30%):** 30 * 0.30 = 9 containers +3. **Total Required Containers:** 30 + 9 = **39 containers** + +### Cold Start Mitigation Strategy +To prevent dropped requests or high latency when spinning up new containers: +1. **Pre-warming (Provisioned Concurrency):** Maintain a buffer of extra "warm" containers running just above the current traffic demands. +2. **Health & Readiness Probes:** Ensure the load balancer only routes traffic to containers that have passed their readiness checks and are fully booted. +3. **Image Optimization:** Keep Docker images small (e.g., using Alpine Linux) so Ghaymah can pull and extract the image rapidly. + +### Using Ghaymah Block Storage for Stateful Workloads +Because containers are ephemeral, stateful workloads (like databases) lose data if a container crashes. Ghaymah Block Storage solves this: +1. **Provision & Mount:** A persistent block storage volume is provisioned and mounted to a specific directory path inside the container +2. **Persistence:** The application writes data directly to this external volume. If the container is destroyed or rescheduled, the volume easily reattaches to the new container, ensuring zero data loss. \ No newline at end of file diff --git a/q5-mithal-monitor/.ghaymah.json b/q5-mithal-monitor/.ghaymah.json new file mode 100644 index 0000000..598f042 --- /dev/null +++ b/q5-mithal-monitor/.ghaymah.json @@ -0,0 +1,17 @@ +{ + "id": "159ccedb-4f26-447c-99e7-bcad1de22120", + "name": "q5-mithal-monitor", + "projectId": "27aacb50-4d56-474c-baae-b52a853b4d57", + "ports": [ + { + "expose": true, + "number": 80 + } + ], + "publicAccess": { + "enabled": true, + "domain": "auto" + }, + "resourceTier": "t1", + "dockerFileName": "Dockerfile" +} diff --git a/q5-mithal-monitor/.github/workflows/ci.yml b/q5-mithal-monitor/.github/workflows/ci.yml new file mode 100644 index 0000000..5825d74 --- /dev/null +++ b/q5-mithal-monitor/.github/workflows/ci.yml @@ -0,0 +1,34 @@ + +name: Ghaymah CI/CD Pipeline + +on: + push: + branches: [ "master" ] + + + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + deploy-production: + name: Deploy to Production + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Install Ghaymah CLI + run: curl -sSL https://cli.ghaymah.systems/install.sh | bash + + - name: Login to Ghaymah + run: $HOME/ghaymah/bin/gy auth login --email "${{ secrets.GHAYMAH_EMAIL }}" --password "${{ secrets.GHAYMAH_PW }}" --debug + + - name: Deploy to Ghaymah (Production) + run: $HOME/ghaymah/bin/gy resource app launch --debug + diff --git a/q5-mithal-monitor/Dockerfile b/q5-mithal-monitor/Dockerfile new file mode 100644 index 0000000..9afb9c6 --- /dev/null +++ b/q5-mithal-monitor/Dockerfile @@ -0,0 +1,28 @@ +FROM nginx:alpine + +# Install bash, curl, openssl, coreutils (for date parsing), and cron +RUN apk update && apk add bash curl openssl coreutils dcron + +# Clear the default Nginx welcome page first +RUN rm -rf /usr/share/nginx/html/* + +# Setup workspace +COPY dashboard.html /usr/share/nginx/html/index.html +COPY monitor.sh /usr/local/bin/monitor.sh +RUN chmod +x /usr/local/bin/monitor.sh + +RUN touch /usr/share/nginx/html/data.csv && \ + chown -R nginx:nginx /usr/share/nginx/html && \ + chmod -R 755 /usr/share/nginx/html + +# Add cron job (runs every minute) +RUN echo "* * * * * /usr/local/bin/monitor.sh" | crontab - + +# Custom entrypoint to start cron in background, then nginx in foreground +RUN echo '#!/bin/sh' > /entrypoint.sh && \ + echo 'crond -b -l 8' >> /entrypoint.sh && \ + echo 'nginx -g "daemon off;"' >> /entrypoint.sh && \ + chmod +x /entrypoint.sh + +EXPOSE 80 +CMD ["/entrypoint.sh"] \ No newline at end of file diff --git a/q5-mithal-monitor/dashboard.html b/q5-mithal-monitor/dashboard.html new file mode 100644 index 0000000..1c2fe4d --- /dev/null +++ b/q5-mithal-monitor/dashboard.html @@ -0,0 +1,115 @@ + + + + + + Mithal Engine Status + + + + + +

Mithal Engine Monitoring Dashboard

+ +

Uptime (Last 24h)

--%

SSL Certificate

-- Days

+ +

Response Time (Last Hour)

+ +

Recent Logs (Last 10 Checks)

+ + + + + + + + + + + +

Timestamp	Status	Latency (s)	DNS (s)	Search (s)

+ + + + \ No newline at end of file diff --git a/q5-mithal-monitor/monitor.sh b/q5-mithal-monitor/monitor.sh new file mode 100644 index 0000000..a620826 --- /dev/null +++ b/q5-mithal-monitor/monitor.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# monitor.sh + +TARGET="mithal.space" +URL="https://$TARGET" +SEARCH_URL="$URL/?q=test" # Adjust query parameter based on the actual search endpoint +if [ -d "/usr/share/nginx/html" ]; then + DATA_FILE="/usr/share/nginx/html/data.csv" +else + DATA_FILE="./data.csv" +fi + + +# Initialize CSV with headers if it doesn't exist +if [ ! -f "$DATA_FILE" ]; then + echo "timestamp,status,latency,ssl_days,dns_time,search_time" > "$DATA_FILE" +fi + +TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +# 1. Uptime, Latency, and DNS Resolution +HTTP_RESPONSE=$(curl -o /dev/null -s -w "%{http_code},%{time_total},%{time_namelookup}" "$URL") +STATUS=$(echo "$HTTP_RESPONSE" | cut -d',' -f1) +LATENCY=$(echo "$HTTP_RESPONSE" | cut -d',' -f2) +DNS_TIME=$(echo "$HTTP_RESPONSE" | cut -d',' -f3) + +# 2. Search Response Time +SEARCH_TIME=$(curl -o /dev/null -s -w "%{time_total}" "$SEARCH_URL") + +# 3. SSL Expiration +EXP_DATE=$(echo | openssl s_client -servername "$TARGET" -connect "$TARGET:443" 2>/dev/null | openssl x509 -noout -enddate | cut -d= -f2) +if [ -n "$EXP_DATE" ]; then + EXP_EPOCH=$(date -d "$EXP_DATE" +%s) + CURRENT_EPOCH=$(date +%s) + SSL_DAYS=$(( (EXP_EPOCH - CURRENT_EPOCH) / 86400 )) +else + SSL_DAYS=0 +fi + +# Append to CSV +echo "$TIMESTAMP,$STATUS,$LATENCY,$SSL_DAYS,$DNS_TIME,$SEARCH_TIME" >> "$DATA_FILE" \ No newline at end of file