commit 8bc563fa6b0af685d3d798701c5765e18fe90ccc Author: yassinelagamy <284392805+yassinelagamy@users.noreply.github.com> Date: Sun Jul 26 19:45:13 2026 +0300 Complete Ghaymah cloud assessment diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..ecbb225 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,101 @@ +name: Build and deploy + +on: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +env: + IMAGE_NAME: docker.io/${{ secrets.DOCKERHUB_USERNAME }}/ghaymah-api + +jobs: + build-push: + name: Build and push image + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push immutable and latest tags + uses: docker/build-push-action@v6 + with: + context: q1-deploy-monitor/app + file: q1-deploy-monitor/app/Dockerfile + push: true + tags: | + ${{ env.IMAGE_NAME }}:${{ github.sha }} + ${{ env.IMAGE_NAME }}:latest + + deploy-staging: + name: Deploy to staging + runs-on: ubuntu-latest + needs: build-push + env: + DEPLOY_IMAGE: docker.io/${{ secrets.DOCKERHUB_USERNAME }}/ghaymah-api:${{ github.sha }} + GHAYMAH_APP_NAME: ${{ vars.GHAYMAH_STAGING_APP }} + GHAYMAH_API_TOKEN: ${{ secrets.GHAYMAH_API_TOKEN }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Show staging deployment target + run: | + echo "Image to deploy: ${DEPLOY_IMAGE}" + echo "Ghaymah app: ${GHAYMAH_APP_NAME:-myapp-staging}" + + - name: Install the documented Ghaymah CLI + run: | + # VERIFY: Official installation command as published at https://ghaymah.systems/docs on 2026-07-26. + curl -sSL https://cli.ghaymah.systems/install.sh | bash + echo "$HOME/ghaymah/bin" >> "$GITHUB_PATH" + + - name: Deploy staging image + shell: bash + run: | + # VERIFY: The CLI is documented, but non-interactive token auth and external-image update syntax are not. Replace this adapter when confirmed. + bash scripts/ghaymah_deploy.sh \ + --app "${GHAYMAH_APP_NAME:-myapp-staging}" \ + --image "${DEPLOY_IMAGE}" + + deploy-production: + name: Deploy to production + runs-on: ubuntu-latest + needs: deploy-staging + environment: production + env: + DEPLOY_IMAGE: docker.io/${{ secrets.DOCKERHUB_USERNAME }}/ghaymah-api:${{ github.sha }} + GHAYMAH_APP_NAME: ${{ vars.GHAYMAH_PRODUCTION_APP }} + GHAYMAH_API_TOKEN: ${{ secrets.GHAYMAH_API_TOKEN }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Show production deployment target + run: | + echo "Image to deploy: ${DEPLOY_IMAGE}" + echo "Ghaymah app: ${GHAYMAH_APP_NAME:-myapp-production}" + + - name: Install the documented Ghaymah CLI + run: | + # VERIFY: Official installation command as published at https://ghaymah.systems/docs on 2026-07-26. + curl -sSL https://cli.ghaymah.systems/install.sh | bash + echo "$HOME/ghaymah/bin" >> "$GITHUB_PATH" + + - name: Deploy production image + shell: bash + run: | + # VERIFY: The CLI is documented, but non-interactive token auth and external-image update syntax are not. Replace this adapter when confirmed. + bash scripts/ghaymah_deploy.sh \ + --app "${GHAYMAH_APP_NAME:-myapp-production}" \ + --image "${DEPLOY_IMAGE}" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..61c40cc --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +# Python +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.venv/ +venv/ + +# Local environment and secrets +.env +.env.* +!.env.example +*.pem +*.key + +# Editors and operating systems +.vscode/ +.idea/ +.DS_Store +Thumbs.db + +# Temporary and log files +*.tmp +*.log + +# Local Ghaymah CLI configuration may contain account or deployment metadata. +.ghaymah.json diff --git a/CHECKLIST.md b/CHECKLIST.md new file mode 100644 index 0000000..dea41c5 --- /dev/null +++ b/CHECKLIST.md @@ -0,0 +1,133 @@ +# Ghaymah Test — Step-by-Step Checklist + +Work through this top to bottom. Each step is tagged with who does it: +**[YOU]** = manual work · **[OPUS]** = prompt Opus 5 · **[SOL]** = prompt Sol/Codex +Steps inside the same phase that have different tags can run at the same time. + +--- + +## PHASE 0 — Setup & Research (Day 1 morning, ~2–3h) + +- [ ] **0.1 [YOU]** Create a Git repo `ghaymah-test` with this folder structure: + ``` + q1-deploy-monitor/{app,monitor,dashboard} + q2-postmortem/ + q3-cicd/ + q4-scalability/ + q5-mithal-dashboard/{collector,data,dashboard} + README.md + ``` +- [ ] **0.2 [YOU]** Create/log in to your **ghaymah.systems** account. +- [ ] **0.3 [YOU]** Explore and take notes + screenshots on: + - [ ] Container platform (how to create a service, set port, get public URL) + - [ ] Container Registry (registry URL, how to log in/push) + - [ ] ghaymah CLI (install command, auth command, deploy command) + - [ ] Monitoring tools (metrics, graphs, alert rules) + - [ ] Block Storage (how volumes attach, limits) + > These notes go into every agent prompt — Q2/Q3/Q4 must cite real Ghaymah features. +- [ ] **0.4 [YOU]** Open **mithal.space** in a browser, run a search, and copy the search URL pattern (e.g. `/search?q=test`). Save it for step 4.1. +- [ ] **0.5 [YOU]** Decide stack: **Python + FastAPI** (assumed below). + +--- + +## PHASE 1 — Dispatch both agents (Day 1 afternoon) + +Send these two prompts at the same time — the tracks don't share files. + +- [ ] **1.1 [OPUS]** Prompt Opus with: full Q1 text + your platform notes + repo layout. Ask for: + - [ ] `q1-deploy-monitor/app/main.py` — FastAPI with `GET /`, `GET /health` (returns `{"status":"ok","uptime_s":...,"timestamp":...}`), `GET /metrics` (in-memory request counter via middleware) + - [ ] `app/requirements.txt` (fastapi, uvicorn) + - [ ] `app/Dockerfile` — `python:3.12-slim`, non-root user, layer-cached COPY order, `EXPOSE 8080`, `HEALTHCHECK` hitting `/health`, plus `.dockerignore` + - [ ] `monitor/monitor.py` — loop every 30s: GET `$APP_URL/health` (5s timeout), append `{"ts","status","code","latency_ms","requests"}` to `monitor/data/checks.json`; "down" on exception; alert print after 3 consecutive fails + - [ ] `dashboard/index.html` — single file, no build step: green/red status badge, latency line chart (Chart.js CDN), request-count tile, fetches `checks.json`, auto-refresh 30s +- [ ] **1.2 [SOL]** Prompt Sol with: full Q2 text + your monitoring-tool notes. Ask for `q2-postmortem/POSTMORTEM.md` containing: + - [ ] Summary (45-min outage, OOMKilled exit 137, impact, severity) + - [ ] Minute-by-minute timeline table (memory creep → OOMKill → crash-loop → alert → limit raise + rollback → recovery) + - [ ] Root cause via 5-Whys (memory leak in release X + undersized limit + no memory alerting/CI check) + - [ ] Recommendations table (owner + priority) + - [ ] Auto-scaling policy: scale out at mem>70% or CPU>65% for 2 min, min 2 replicas, scale-in cooldown 5–10 min — **and** an explicit note that HPA alone doesn't fix a leak + - [ ] Early-detection section citing real Ghaymah monitoring features; every platform claim marked `` + +--- + +## PHASE 2 — Q1 goes live (Day 2 morning) + +- [ ] **2.1 [YOU]** Review Opus's Q1 output. Build & test locally: + ``` + docker build -t ghaymah-api ./q1-deploy-monitor/app + docker run -p 8080:8080 ghaymah-api + ``` + - [ ] `http://localhost:8080/health` returns 200 JSON + - [ ] `/metrics` counter increments +- [ ] **2.2 [YOU]** Push image to ghaymah Container Registry (`docker login` → `docker tag` → `docker push`). +- [ ] **2.3 [YOU]** Create the container service on ghaymah.systems (port 8080), note the **public URL**, screenshot the running service + `/health` response. +- [ ] **2.4 [YOU]** Start `monitor.py` with `APP_URL=` and **leave it running** so real data accumulates before submission. +- [ ] **2.5 [YOU]** Open the Q1 dashboard against the growing `checks.json` — verify badge, chart, counter all render. +- [ ] **2.6 [OPUS]** Send any deployment fixes back to Opus, then immediately give it Q5 (step 4.1) — don't wait. + +--- + +## PHASE 3 — CI/CD (Day 2 morning, parallel with Phase 2) + +- [ ] **3.1 [SOL]** Prompt Sol with: full Q3 text + your CLI/registry notes. Ask for: + - [ ] `.github/workflows/deploy.yml` — trigger on push to `main` + `workflow_dispatch`; jobs: + - `build-push`: checkout → `docker/login-action` (secrets `GHAYMAH_REGISTRY_USER`/`TOKEN`) → `docker/build-push-action` tagging `:${{ github.sha }}` and `:latest` + - `deploy-staging` (needs build-push): deploy via ghaymah CLI + - `deploy-production` (needs staging): bound to GitHub Environment `production` (this is the manual-approval gate) + - [ ] `q3-cicd/CICD.md` — staging vs production section (data, scale, secrets, access, alerting, cadence + promotion flow diagram) and ghaymah CLI integration section (install, `auth login` with `GHAYMAH_API_TOKEN` secret, deploy command); placeholders marked `` +- [ ] **3.2 [YOU]** Push repo to GitHub. In repo settings: + - [ ] Add secrets: `GHAYMAH_REGISTRY_USER`, `GHAYMAH_REGISTRY_TOKEN`, `GHAYMAH_API_TOKEN` + - [ ] Settings → Environments → create `production` → add yourself as **required reviewer** +- [ ] **3.3 [YOU]** Trigger the workflow, let it pause at the production gate, **screenshot the approval prompt**, then approve. + +--- + +## PHASE 4 — Q5 build & deploy (Day 2 afternoon) + +- [ ] **4.1 [OPUS]** Prompt Opus with: full Q5 text + the mithal.space search URL from 0.4. Ask for: + - [ ] `collector/collect.py` — every 60s (plus `--once` flag) measure: + - Latency: timed `GET https://mithal.space` + - Uptime: up if status 200–399 + - SSL: `ssl`/`socket` cert `notAfter` → days remaining + - DNS: timed `socket.getaddrinfo` + - Search: timed request to the search URL (configurable) + - Append `{"ts","up","code","latency_ms","dns_ms","ssl_days_left","search_ms"}` to `data/metrics.json`, prune >48h + - [ ] `dashboard/index.html` — uptime % tile (24h), latency line chart (last hour, Chart.js), SSL card color-coded (green>30d / yellow 8–30 / red≤7), last-10-checks table, auto-refresh 60s; same visual style as Q1 + - [ ] `Dockerfile` — one container serving the dashboard statically **and** running the collector in the background (shell entrypoint launching both), collector writes into the served `data/` dir +- [ ] **4.2 [YOU]** Test locally (`--once` first, then the loop; open the dashboard). Fix issues via Opus. +- [ ] **4.3 [YOU]** Push image + deploy to ghaymah exactly like 2.2–2.3. Screenshot the live dashboard URL. +- [ ] **4.4 [YOU]** Leave it running so the 24h uptime % and hourly chart fill with real data. + +--- + +## PHASE 5 — Q4 scalability doc (Day 2 afternoon, parallel with Phase 4) + +- [ ] **5.1 [SOL]** Prompt Sol with: full Q4 text + your Block Storage notes. Ask for `q4-scalability/SCALABILITY.md`: + - [ ] Mermaid architecture diagram: DNS → CDN → L7 load balancer → stateless API fleet (auto-scaled) → Redis cache → DB primary+replicas on Block Storage; monitoring on the side; annotate the 15k req/s path + - [ ] Container math shown explicitly: `15,000 ÷ 500 = 30 → ×1.30 = 39 containers`, plus notes on rounding up and N+1 for rolling deploys + - [ ] Cold-start strategy: warm pool / min replicas, slim pre-pulled images, readiness-probe gating, predictive scaling before peaks, step scale-out policies + - [ ] Block Storage section: ephemeral containers vs persistent volumes, one-writer semantics, snapshots/backups, IOPS; API tier stays diskless — `` markers on platform specifics + +--- + +## PHASE 6 — Verification & polish (Day 3 morning) + +- [ ] **6.1 [YOU]** Search the whole repo for `` and resolve every one against real Ghaymah docs (fix CLI commands, registry URL, monitoring feature names, Block Storage details). +- [ ] **6.2 [YOU]** Insert real values everywhere: live app URL (Q1), registry image name (Q3), live dashboard URL (Q5). +- [ ] **6.3 [YOU]** Add all screenshots to the repo: Q1 service + `/health`, Q3 approval gate, Q5 live dashboard. +- [ ] **6.4 [SOL]** Send Sol any corrections needed in Q2/Q3/Q4 docs from your verification pass. +- [ ] **6.5 [OPUS]** Ask Opus for final dashboard polish now that real data exists (empty-state handling, chart scaling). + +--- + +## PHASE 7 — Final review & submit (Day 3 afternoon) + +- [ ] **7.1 [YOU]** Rubric pass — check every numbered sub-requirement one by one: + - [ ] Q1: Dockerfile · deployed on ghaymah · `/health` · 30s monitor script · dashboard (status, latency, request count) + - [ ] Q2: postmortem (summary, timeline, root cause, recommendations) · auto-scaling policy · early-detection with Ghaymah tools + - [ ] Q3: build+push workflow · manual approval before production · staging vs production explained · CLI integration documented + - [ ] Q4: architecture diagram · container count (39) with math · cold-start strategy · Block Storage explanation + - [ ] Q5: collector (latency, uptime, SSL, DNS, search) every minute · CSV/JSON storage · dashboard (uptime %, latency chart, SSL, last 10 checks) · deployed on ghaymah +- [ ] **7.2 [YOU]** Write `README.md` as an index: one section per question, links to every file, live URLs, screenshots. +- [ ] **7.3 [YOU]** Confirm both dashboards show real accumulated data (not empty charts). +- [ ] **7.4 [YOU]** Final commit + push. Submit. diff --git a/FINALIZE.md b/FINALIZE.md new file mode 100644 index 0000000..dcf6592 --- /dev/null +++ b/FINALIZE.md @@ -0,0 +1,228 @@ +# Finalization Runbook + +Everything that can be done without your credentials is done. This file is the +copy-paste path through what remains. Work top to bottom. + +> Note: written while another agent (Codex) was editing files in this repo. The +> line numbers in §5 are a snapshot — re-run the grep before trusting them. + +--- + +## 0. What is already verified (no need to redo) + +Verified locally on 2026-07-26 with Docker 29.5.3 (linux/amd64): + +| Item | Result | +|---|---| +| `docker build` Q1 (`q1-deploy-monitor/app`) | builds clean → `ghaymah-api:latest`, 249 MB | +| `docker build` Q5 (`q5-mithal-dashboard`) | builds clean → `mithal-monitor:latest`, 200 MB | +| Q1 container | `/health` → 200 `{"status":"ok","uptime_s":11.254,...}`; `/metrics` counter increments; Docker healthcheck reports **healthy** | +| Q5 container | `start.sh` launches collector + static server; collector wrote a real record inside the container (`up:true, code:200, ssl_days_left:50`); dashboard rendered at `/index.html` with badge, uptime tile, TLS card, chart and table; healthcheck **healthy** | +| Q1 monitor | up/down transitions, `ALERT` after exactly 3 consecutive failures, `--once` exit codes | +| Q5 collector | live measurements against mithal.space; unreachable-host run records all-nulls without crashing | + +**Both images are already built on this machine.** You can go straight to `docker push`. + +--- + +## 1. Set your Docker Hub username once + +Everything below reuses this. Run in PowerShell: + +```powershell +$DH = "yourdockerhubusername" # <-- edit this line only +``` + +--- + +## 2. Push both images (10 min) + +```powershell +docker login +``` + +```powershell +docker tag ghaymah-api:latest "docker.io/$DH/ghaymah-api:latest" +docker tag mithal-monitor:latest "docker.io/$DH/mithal-monitor:latest" +docker push "docker.io/$DH/ghaymah-api:latest" +docker push "docker.io/$DH/mithal-monitor:latest" +``` + +Then, on hub.docker.com, confirm **both repositories are Public** — Ghaymah pulls +them anonymously. A private repo is the single most common cause of a deployment +that never leaves "pulling". + +--- + +## 3. Deploy both apps on Ghaymah (30 min) + +Same form twice: + +| Field | Q1 | Q5 | +|---|---|---| +| Container Image URL | `docker.io//ghaymah-api:latest` | `docker.io//mithal-monitor:latest` | +| Application Name | `ghaymah-api` | `mithal-monitor` | +| Port Number | `8080` | `8080` | +| Public Access | enabled | enabled | +| Env vars (optional) | `APP_NAME=ghaymah-api`, `APP_VERSION=1.0.0` | `INTERVAL_S=60` | + +Record both public URLs — you need them in §5. + +Verify immediately: + +```powershell +curl.exe -f "/health" +``` + +Then open `/index.html` in a browser (it will show 1–2 checks at first). + +### Start the Q1 monitor the moment Q1 is live + +This is the long pole — the Q1 dashboard is only convincing with hours of history. +Leave this window open for the rest of the assessment: + +```powershell +$env:APP_URL=""; python q1-deploy-monitor\monitor\monitor.py +``` + +Q5 needs nothing — its collector runs inside the deployed container. + +--- + +## 4. Resolve the Q3 blocker (the CLI question) — 20 min + +**What I established from public sources** (so you don't repeat the search): + +- Install: `curl -sSL https://cli.ghaymah.systems/install.sh | bash` +- The installer downloads `gy-{platform}-{arch}` from `https://cli.ghaymah.systems` + (override with `SERVER_URL`) and installs to `$HOME/ghaymah/bin/gy`. +- The only env var the installer honours is `SERVER_URL`. **No token env var exists + in the installer.** +- Documented commands: `gy auth login` (browser-based), `gy auth status`, + `gy resource project get`, `gy resource project create --set .name=my-new-project`, + `gy resource application init --project-id `, `gy resource application launch`, + `gy resource application logs`. +- `.ghaymah.json` fields: `id`, `name`, `projectId`, `ports[].expose/.number`, + `publicAccess.enabled/.domain`, `resourceTier`, `dockerFileName`, `env`, `domains`. +- **Nothing public documents non-interactive auth, a registry hostname, or updating + an application to an external image tag.** `/docs/cli` returns 404; the changelog + and product pages list "container registry" and "CI/CD" as features with no syntax. + +**Run these three commands after `gy auth login`** — they are the fastest way to +settle it, and their output is what I need to finish the workflow: + +```bash +gy auth login --help +gy --help +gy resource application --help +``` + +Look specifically for: a `--token` / `--api-key` flag, any `GHAYMAH_*` env var, a +`gy resource application update`/`set-image` subcommand, or a `gy registry` group. +Also check the authenticated dashboard for an **API tokens / service accounts** +page and for the registry hostname. + +Paste whatever those print back to me and I will finish `deploy.yml`, +`scripts/ghaymah_deploy.sh` and `CICD.md` in one pass. + +If it turns out no non-interactive path exists, that is a legitimate finding — +the honest write-up (deploy job documented, adapter exits 78 rather than faking +success, manual promotion step described) scores better than invented syntax. +Do **not** invent a command to make the pipeline look green. + +--- + +## 5. Replace the placeholders + +Snapshot of what is unfilled (re-run the grep after Codex finishes): + +```powershell +Select-String -Path (Get-ChildItem -Recurse -Include *.md,*.yml,*.sh -Path . | Where-Object FullName -notmatch '\\\.git\\') -Pattern 'MY_DOCKERHUB_USER|the public URL Ghaymah assigns|` × 5 | `q2-postmortem/POSTMORTEM.md:133`, `q3-cicd/CICD.md:111,125`, `q4-scalability/SCALABILITY.md:108,112` | see §6 | + +Bulk replacement (run **after** Codex is done, from the repo root): + +```powershell +$DH="yourdockerhubusername"; $Q1="https://your-q1-url"; $Q5="https://your-q5-url" +Get-ChildItem -Recurse -Include *.md -File | Where-Object FullName -notmatch '\\\.git\\' | ForEach-Object { + (Get-Content $_.FullName -Raw) ` + -replace '', $DH ` + -replace '', $Q1 | + Set-Content $_.FullName -Encoding utf8 +} +``` + +That sets the Q1 URL everywhere; then fix the single Q5 occurrence in +`q5-mithal-dashboard/README.md` by hand (it is the `/index.html` line). + +--- + +## 6. The five remaining VERIFY areas + +Each needs one look at the **authenticated** dashboard. What to check, and what to +do with the answer: + +| # | File | Question to answer in the dashboard | +|---|---|---| +| 1 | `q2-postmortem/POSTMORTEM.md:133` | Does Ghaymah show per-container memory/CPU metrics, restart counts, and configurable alert rules + notification channels? Screenshot the monitoring page. | +| 2 | `q3-cicd/CICD.md:111` | Non-interactive auth for CI — settled by §4. | +| 3 | `q3-cicd/CICD.md:125` | External-image update syntax — settled by §4. | +| 4 | `q4-scalability/SCALABILITY.md:108` | Block Storage: snapshots supported? retention/encryption/restore? | +| 5 | `q4-scalability/SCALABILITY.md:112` | Block Storage: size limits, resize, IOPS/throughput, attachment/access modes, zone binding. | + +Rule for all five: if the platform confirms it, state it plainly and drop the +marker. If the platform does **not** expose it, rewrite the sentence to describe +the general requirement and say the platform-specific limits were not documented +at the time of writing — then drop the marker. **No `VERIFY` string may survive in +the submitted repo.** + +--- + +## 7. Screenshots to capture + +Save under `docs/screenshots/` with these names so the READMEs can link them: + +- `q1-service-running.png` — the Ghaymah dashboard showing the running Q1 app +- `q1-health-response.png` — browser or terminal showing `/health` → 200 JSON +- `q1-monitor-dashboard.png` — the Q1 dashboard with a populated latency chart +- `q3-approval-gate.png` — the GitHub Actions run paused on the production reviewer prompt +- `q5-live-dashboard.png` — the deployed Q5 dashboard with ≥ 1 h of history +- `ghaymah-monitoring.png` — optional, supports the Q2 detection section + +--- + +## 8. GitHub, secrets, pipeline + +1. Create the repo, add the remote, push. +2. Settings → Secrets and variables → Actions: add the secrets named in + `.github/workflows/deploy.yml` (Docker Hub user/token, plus whatever §4 settles + for Ghaymah). +3. Settings → Environments → `production` → **Required reviewers: yourself**. +4. Run the workflow → let it pause → screenshot the gate → approve. + +Decide before pushing whether `PLAN.md`, `PROMPTS.md` and `CHECKLIST.md` belong in +the submission. They document your internal agent workflow; graders don't need +them. Either delete them or list them in `.gitignore`. + +--- + +## 9. Final gate before submitting + +```powershell +Select-String -Path (Get-ChildItem -Recurse -Include *.md,*.yml,*.sh -File | Where-Object FullName -notmatch '\\\.git\\') -Pattern 'VERIFY|MY_DOCKERHUB_USER|the public URL Ghaymah assigns' +``` + +Must return nothing. Then: + +- [ ] Both apps reachable at their public URLs +- [ ] Q1 dashboard chart has hours of points; Q5 uptime tile computed over real 24 h data +- [ ] All screenshots committed and linked from the root README +- [ ] Every numbered sub-requirement of Q1–Q5 ticked (CHECKLIST.md §7.1) +- [ ] Final commit pushed diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..155c617 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,218 @@ +# Ghaymah Internship Test — Implementation Plan + +**Scope:** 5 questions · 72 hours · 100 points (20 each) +**Execution model:** Two agents working in parallel — **Sol** (Codex) and **Opus 5** — with you as the integrator who does the manual platform steps (accounts, deployments, secrets). + +--- + +## 0. Ground Rules & Repo Setup (do this first, yourself — 30 min) + +**Purpose:** Both agents need a shared structure so their outputs merge cleanly without conflicts. Doing this before dispatching work prevents rework. + +**How:** +1. Create one Git repo (e.g. `ghaymah-test`) with this layout: + +``` +ghaymah-test/ +├── q1-deploy-monitor/ +│ ├── app/ # API + Dockerfile +│ ├── monitor/ # monitoring script +│ └── dashboard/ # HTML/CSS/JS dashboard +├── q2-postmortem/ +│ └── POSTMORTEM.md +├── q3-cicd/ +│ ├── .github/workflows/ # (copied to repo root at the end) +│ └── CICD.md +├── q4-scalability/ +│ └── SCALABILITY.md # diagram + math + strategy +├── q5-mithal-dashboard/ +│ ├── collector/ # metrics script +│ ├── data/ # CSV/JSON output +│ └── dashboard/ +└── README.md # index of all answers +``` + +2. Sign up / log in to **ghaymah.systems**, explore: Container platform, Container Registry, CLI docs, monitoring tools, Block Storage docs. **Take notes/screenshots** — Q2, Q3, Q4 answers must reference *real* Ghaymah features, not generic cloud talk. This research is the single highest-value manual task: graders will notice platform-specific accuracy. +3. Decide the stack once: **Python + FastAPI** for both APIs (Q1 and Q5 collector share language, smaller surface area) — or Node/Express if you prefer. The plan below assumes Python. + +**Why manual:** account creation, credentials, and platform exploration cannot be delegated to agents (and shouldn't be — never give agents your passwords/API keys; store tokens as env vars/GitHub secrets yourself). + +--- + +## Parallelization Map (who does what) + +| Track | Agent | Work | Rationale | +|---|---|---|---| +| A | **Opus 5** | Q1 (app + Dockerfile + monitor + dashboard), Q5 (collector + dashboard) | Heaviest coding volume, frontend polish, end-to-end coherence — Opus is strongest at multi-file builds and design-quality dashboards | +| B | **Sol** | Q3 (CI/CD workflow + docs), Q2 (postmortem), Q4 (scaling doc + diagram) | Mostly structured writing + one YAML workflow — well-bounded, spec-driven tasks that run independently of Track A | +| — | **You** | Ghaymah account, deployments, registry creds, GitHub secrets, real screenshots, final review | Anything requiring credentials or the live platform | + +The two tracks share **zero files**, so they can run truly simultaneously. Merge point: Day 2 evening, when you deploy Track A's artifacts and paste real values (URLs, registry names) into Track B's docs. + +**Prompting tip:** give each agent the full question text *plus* your Ghaymah platform notes from step 0, and tell them the repo layout above so paths match. + +--- + +## Q1 — Deploy & Monitor an App on Ghaymah (20 pts) + +### 1.1 The API app + `/health` endpoint — **Opus** +**Purpose:** The deliverable everything else in Q1 hangs off. `/health` is the standard liveness contract that orchestrators and your monitor script both consume. + +**How:** Small FastAPI app with: +- `GET /` — hello/info route +- `GET /health` — returns `{"status":"ok","uptime_s":...,"timestamp":...}` with HTTP 200. Keep it dependency-free (no DB check) so it reflects process liveness only. +- `GET /metrics` — returns an in-memory request counter + simple stats (this feeds the dashboard's "عدد الطلبات" requirement). Implement with a middleware that increments a counter per request. + +**Implementation:** `app/main.py`, `app/requirements.txt` (fastapi, uvicorn). Run with `uvicorn main:app --host 0.0.0.0 --port 8080`. + +### 1.2 Dockerfile — **Opus** +**Purpose:** Shows you understand image hygiene, not just "it runs". + +**How:** Multi-stage or slim single-stage: +- `python:3.12-slim` base, non-root user, `COPY requirements.txt` before code (layer caching), `EXPOSE 8080`, `HEALTHCHECK CMD curl -f http://localhost:8080/health || exit 1`, `CMD uvicorn...`. +- Add `.dockerignore`. + +**Implementation check:** you build locally: `docker build -t ghaymah-api . && docker run -p 8080:8080 ghaymah-api`, hit `/health`. + +### 1.3 Deploy to ghaymah.systems — **You (manual)** +**Purpose:** The 20 points require a *live* deployment; agents can't do this. + +**How:** Per platform docs — typically: push image to ghaymah Container Registry (`docker login `, `docker tag`, `docker push`), then create a container service in the dashboard/CLI pointing at the image, set port 8080, note the public URL. **Screenshot the running service and the `/health` response** for your submission. + +### 1.4 Monitoring script (every 30s) — **Opus** +**Purpose:** Demonstrates operational thinking — a poller that produces machine-readable history the dashboard can consume. + +**How:** `monitor/monitor.py`: +- Loop: every 30s, `GET /health` with timeout (e.g. 5s), record `timestamp, status ("up"/"down"), http_code, response_ms`; also pull `/metrics` for request count. +- Append each check to `monitor/data/checks.json` (or JSONL) — this file **is the dashboard's data source**, so agree the schema up front: + ```json + {"ts":"2026-07-26T12:00:00Z","status":"up","code":200,"latency_ms":42,"requests":1337} + ``` +- Config via env var `APP_URL`. Handle exceptions → status "down", latency null. Optional: alert line to stdout when 3 consecutive failures. + +### 1.5 Dashboard (HTML/CSS/JS) — **Opus** +**Purpose:** Shows the three required metrics: status, response time, request count. + +**How:** Single static page `dashboard/index.html` (inline CSS/JS, zero build step): +- Status badge (green/red) from latest check +- Line chart of `latency_ms` (Chart.js from CDN, or hand-rolled SVG if you want zero dependencies) +- Request counter tile +- Fetches `checks.json` (served next to it, or the monitor writes into the dashboard folder) and refreshes every 30s with `setInterval`. + +**Deliverable checklist Q1:** Dockerfile ✔ live URL ✔ `/health` ✔ monitor script ✔ dashboard ✔ screenshots ✔ + +--- + +## Q2 — OOMKilled Postmortem (20 pts) — **Sol** + +**Purpose:** Pure documentation question testing incident-response maturity. No code; graded on structure, realism, and platform-specific recommendations. + +**How:** `q2-postmortem/POSTMORTEM.md` in the classic blameless format: + +1. **Summary** — 45-min outage, repeated OOMKilled restarts (exit code 137), impact (error rate / downtime %), severity level. +2. **Timeline** — invent a realistic minute-by-minute table: memory creep begins → first OOMKill → crash-loop (each restart re-accumulates memory faster under retry traffic) → alert fires → engineer raises memory limit + rolls back the leaking release → recovery. Timestamps, actor, action. +3. **Root cause** — pick something concrete and defensible, e.g. a memory leak introduced in release X (unbounded in-process cache) combined with a memory limit sized for the old baseline; 5-Whys chain down to "no memory regression check in CI, no memory alerting." +4. **Recommendations** — table with owner + priority: right-size limits, fix leak, add memory alerts at 80%, add auto-scaling (links to part 2), load-test with soak tests, add runbook. +5. **Auto-scaling policy design** — a concrete policy for Ghaymah containers: + - Horizontal: scale out at avg memory > 70% or CPU > 65% for 2 min; min 2 / max N replicas; scale-in cooldown 5–10 min to avoid flapping. + - Explicitly explain *why HPA alone doesn't fix a leak* (leaks eat any replica count — scaling buys time for the fix, plus restart policy / vertical headroom). This nuance is what separates a top answer. +6. **Early detection with Ghaymah monitoring** — reference the actual tools you found in step 0: memory-usage graphs per container, alert rules (memory > 80% for 5 min, restart-count > 3/10 min, OOMKilled event alerts), dashboards + notification channels. + +**Implementation:** one well-formatted Markdown file; optionally a small Mermaid timeline/graph. **You** later swap in accurate Ghaymah tool names from your notes. + +--- + +## Q3 — CI/CD Pipeline (20 pts) — **Sol** + +**Purpose:** Tests GitHub Actions fluency + registry integration + release discipline (manual gate). + +### 3.1 Workflow — build & push to ghaymah Container Registry +**How:** `.github/workflows/deploy.yml`: +- Trigger: `push` to `main` (+ `workflow_dispatch`). +- Job `build-push`: checkout → `docker/login-action` against the ghaymah registry using `secrets.GHAYMAH_REGISTRY_USER/TOKEN` → `docker/build-push-action` tagging `registry.ghaymah.systems//app:${{ github.sha }}` and `:latest`. +- Job `deploy-staging` (needs build): deploy via ghaymah CLI to the staging service. +- Job `deploy-production` (needs staging): bound to a GitHub **Environment** named `production` with **required reviewers** — this is the manual-approval mechanism (job pauses until a human approves in the Actions UI). Sol should document that the environment + reviewers are configured in repo Settings → Environments, since that part isn't in YAML. + +### 3.2 Staging vs Production explanation +**How:** Section in `CICD.md`: purpose (validation vs. live users), differences (data, scale/replicas, secrets, access control, alerting thresholds, deploy cadence), promotion flow diagram `commit → build → staging (auto) → approval → production`. + +### 3.3 ghaymah CLI integration docs +**How:** `CICD.md` section: install CLI, `ghaymah auth login` with an API token stored as `GHAYMAH_API_TOKEN` GitHub secret, deploy/update-image command, how the workflow calls it in a step. **You** verify exact CLI command names from real docs and correct Sol's draft — flag every placeholder Sol writes with `` so nothing invented ships. + +**You (manual):** create the GitHub repo, add the secrets, create the `production` environment with yourself as reviewer, run the pipeline once and screenshot the approval gate. + +--- + +## Q4 — Scalability & Load Balancing (20 pts) — **Sol** + +**Purpose:** Architecture reasoning + arithmetic + platform storage knowledge. Fully deterministic — ideal Sol task. + +### 4.1 Architecture diagram (15,000 req/s) +**How:** Mermaid diagram in `SCALABILITY.md` (renders on GitHub): DNS → CDN/edge cache → Load Balancer (L7) → stateless API container fleet (auto-scaled) → cache layer (Redis) → DB (primary + replicas) → Block Storage for stateful pieces; plus monitoring/queue components. Annotate where the 15k req/s flows and what absorbs bursts. + +### 4.2 Container count math — show the work +``` +15,000 req/s ÷ 500 req/s per container = 30 containers ++30% headroom: 30 × 1.30 = 39 containers +``` +State the answer plainly (**39**), then add operational notes: round up, N+1 for rolling deploys, and that headroom covers spikes + AZ loss. + +### 4.3 Cold-start strategy +**How:** bullet strategy: keep a warm pool / min-replicas floor, pre-pull & slim images (small base, fewer layers), lazy-load nothing critical at boot, readiness probe gating so LB never routes to a cold container, predictive/scheduled scaling ahead of known peaks, gradual (step) scale-out policies. + +### 4.4 ghaymah Block Storage for stateful workloads +**How:** explain: containers are ephemeral → attach Block Storage volumes for databases/queues/uploads; persistence across restarts/reschedules; one-writer-per-volume semantics (so stateful services scale differently than stateless API tier); snapshots/backups; IOPS considerations. Tie back to the diagram (DB nodes mount Block Storage; API tier stays diskless). **You** correct against real Ghaymah Block Storage docs. + +--- + +## Q5 — mithal.space Monitoring Dashboard (20 pts) — **Opus** + +**Purpose:** The most integrated build: multi-metric collector + persisted history + richer dashboard + deployment. Assign to Opus because it shares patterns (and dashboard code style) with Q1 — one agent keeps them consistent. + +### 5.1 Collector script (every minute) — `collector/collect.py` +**How (one Python script, stdlib + `requests`):** +- **Latency:** `requests.get("https://mithal.space", timeout=10)` — measure elapsed ms. +- **Uptime:** same request's status code → up if `200 ≤ code < 400`. +- **SSL:** `ssl` + `socket` — open TLS connection to port 443, read cert `notAfter`, compute days remaining. +- **DNS:** time `socket.getaddrinfo("mithal.space", 443)` in ms. +- **Search response:** since mithal.space is a search engine — send a query (inspect the site first to find the search URL pattern, e.g. `/search?q=test` or its API endpoint) and time the response. **You** should check the actual URL format in a browser and give it to Opus; have Opus make it configurable. +- Scheduling: `while True: run(); sleep(60)` **plus** support one-shot mode (`--once`) so it can also run under cron / a scheduler. + +### 5.2 Storage +**How:** append one JSON object per check to `data/metrics.json` (array) or JSONL; keep a rolling window (e.g. last 24–48h, prune older) so the file stays small. Schema fixed up-front (same discipline as Q1): +```json +{"ts": "...", "up": true, "code": 200, "latency_ms": 120, + "dns_ms": 18, "ssl_days_left": 143, "search_ms": 210} +``` + +### 5.3 Dashboard — `dashboard/index.html` +**How:** static page, fetches `metrics.json`, renders: +- **Uptime % (24h):** `checks_up / checks_total * 100` over last 24h — big number tile. +- **Latency line chart (last hour):** Chart.js line of `latency_ms` (optionally overlay `search_ms`). +- **SSL card:** days remaining, color-coded (green > 30, yellow 8–30, red ≤ 7). +- **Last-10-checks table:** time, status ✅/❌, code, latency, DNS, search. +- Auto-refresh every 60s. Same visual language as Q1's dashboard (Opus keeps them consistent). + +### 5.4 Deploy dashboard to ghaymah — **You + Opus** +**How:** Opus writes a tiny Dockerfile that serves the dashboard **and** runs the collector in the same container (simplest: FastAPI/nginx serving static files + collector as background process writing into the served `data/` dir — a small `supervisord` or a shell entrypoint launching both). You push and deploy exactly as in Q1, screenshot the live URL. + +--- + +## Timeline (72h budget, comfortably front-loaded) + +| When | You | Opus (Track A) | Sol (Track B) | +|---|---|---|---| +| **Day 1 AM** | Repo setup, Ghaymah account + docs research, notes | — | — | +| **Day 1 PM** | Feed notes + prompts to both agents | Q1 app, Dockerfile, monitor, dashboard | Q2 postmortem full draft | +| **Day 2 AM** | Build & deploy Q1, registry setup, GitHub secrets | Q5 collector + dashboard | Q3 workflow + CICD.md | +| **Day 2 PM** | Deploy Q5, run pipeline, capture approval screenshot | Fix anything found in deployment | Q4 scalability doc | +| **Day 3 AM** | Verify every `` against real Ghaymah docs, insert real URLs/screenshots | Polish dashboards with real data | Revisions from your review | +| **Day 3 PM** | Final README index, cross-check rubric (all sub-items × 5 questions), submit | — | — | + +## Quality Gates (your final pass) + +1. Every question's *numbered sub-requirements* are individually answered — graders score per item. +2. No invented Ghaymah feature names survive — everything platform-specific is verified against real docs. +3. Both apps are **live** with URLs + screenshots in the README. +4. Monitor/collector scripts actually ran long enough to produce real data in the dashboards (start them Day 2 so charts aren't empty at submission). +5. Repo README links every deliverable — one-click grading experience. diff --git a/PROMPTS.md b/PROMPTS.md new file mode 100644 index 0000000..083d1ef --- /dev/null +++ b/PROMPTS.md @@ -0,0 +1,223 @@ +# Copy-Paste Agent Prompts + Ghaymah Info to Collect + +> **PART C (added after research): the actual notes to paste — see bottom of this file. Docs turned out sparse, so Part C replaces Part A.** + +## PART A — What to grab from ghaymah.systems FIRST (~20 min) + +Log in and write down these exact values. Paste them into the `[GHAYMAH NOTES]` block of each prompt below. + +1. **Registry URL** — the container registry hostname (e.g. `registry.ghaymah.systems` or similar) and how login works (username + token? access key?). +2. **How to deploy a container** — dashboard steps or CLI command to create a service from an image, set the port, and get the public URL. Note what the public URLs look like. +3. **CLI** — exact install command, exact auth/login command, exact deploy/update command (copy them verbatim from the docs page). +4. **Monitoring** — what metrics/graphs exist per container (CPU? memory? restarts?), whether alert rules exist and what they can trigger on, notification channels. +5. **Auto-scaling** — does the platform have built-in autoscaling settings? What knobs (min/max replicas, CPU/memory thresholds)? +6. **Block Storage** — product name as they call it, how a volume attaches to a container, size limits, snapshot support. +7. **Environments** — can you run two services (staging + production)? Any built-in env concept, or just two separate services? +8. Also: open **mithal.space**, do a search, copy the result URL pattern (e.g. `https://mithal.space/search?q=test`). + +Screenshot everything as you go — you need screenshots for the submission anyway. + +--- + +## PART B — The 5 prompts + +Fill `[GHAYMAH NOTES: ...]` with the relevant items from Part A before sending. Send Prompt 1 (Opus) and Prompt 2 (Sol) simultaneously; then 3 & 4; then 5. + +--- + +### PROMPT 1 → OPUS (Q1: app + Dockerfile + monitor + dashboard) + +``` +Build a complete deliverable for this task, in the folder q1-deploy-monitor/: + +TASK: Deploy and monitor a simple API app on the Ghaymah cloud container platform. + +Create exactly these files: + +1. app/main.py — Python FastAPI app with: + - GET / → basic info JSON + - GET /health → {"status":"ok","uptime_s":,"timestamp":} with HTTP 200, no external dependencies + - GET /metrics → {"requests_total":, "started_at":} using an in-memory counter incremented by a middleware on every request + - Runs with uvicorn on 0.0.0.0:8080 + +2. app/requirements.txt — fastapi + uvicorn, pinned versions. + +3. app/Dockerfile — python:3.12-slim, non-root user, COPY requirements.txt and pip install BEFORE copying code (layer caching), EXPOSE 8080, HEALTHCHECK curl -f http://localhost:8080/health, CMD uvicorn. Also app/.dockerignore. + +4. monitor/monitor.py — Python monitoring script: + - Reads APP_URL from env var + - Every 30 seconds: GET $APP_URL/health with 5s timeout, and GET /metrics + - Appends one JSON object per check to monitor/data/checks.json (a JSON array): + {"ts":"","status":"up"|"down","code":,"latency_ms":,"requests":} + - Any exception/timeout → status "down" + - Prints an ALERT line after 3 consecutive failures + - Also supports --once flag for a single check + +5. dashboard/index.html — ONE self-contained file (inline CSS/JS, Chart.js from CDN allowed) that fetches ../monitor/data/checks.json (path configurable in a const at top) and shows: + - Big status badge, green UP / red DOWN, from the latest check + - Line chart of latency_ms over time + - Request-count tile from the latest check + - Last-updated timestamp, auto-refresh every 30s + - Handle empty/missing data gracefully (show "no data yet", don't crash) + Make it look clean and professional (dark theme is fine). + +Also write q1-deploy-monitor/README.md: how to build/run the Docker image locally, how to push it to the Ghaymah registry and deploy it, how to run the monitor, how to open the dashboard. + +[GHAYMAH NOTES: ] +``` + +--- + +### PROMPT 2 → SOL (Q2: postmortem) + +``` +Write q2-postmortem/POSTMORTEM.md — a professional, blameless postmortem for this scenario: + +SCENARIO: An application on the Ghaymah cloud platform was down for 45 minutes due to repeated OOMKilled container restarts. + +Required sections: + +1. Summary — one paragraph: duration 45 min, cause OOMKilled (exit code 137) crash loop, user impact (est. error rate / full downtime), severity SEV-2. + +2. Timeline — a table with minute-by-minute realistic entries (invent plausible timestamps): gradual memory growth after a release → first OOMKill → crash loop worsened by retry traffic → alert fires → engineer investigates → mitigation (memory limit raised + rollback of the leaking release) → full recovery at minute 45. Columns: time, event, actor/action. + +3. Root cause — 5-Whys analysis. Root: a memory leak introduced in the latest release (unbounded in-process cache), combined with a memory limit sized for the old baseline, no memory-usage alerting, and no memory regression testing in CI. + +4. Recommendations — table with columns: action, owner (role), priority (P0/P1/P2). Include: fix the leak, right-size memory limits, memory alert at 80% for 5 min, restart-count alert, add auto-scaling, soak/load tests in CI, write a runbook. + +5. Auto-scaling policy for Ghaymah — a concrete policy spec: horizontal scaling, scale OUT when avg memory > 70% OR CPU > 65% sustained 2 min; min 2 replicas, max N; scale-in cooldown 5–10 min to prevent flapping. Then an explicit honest paragraph: why autoscaling alone does NOT fix a memory leak (every replica leaks; scaling buys time and absorbs the retry storm while the leak is fixed) — plus vertical headroom and restart policies as complements. + +6. Early detection with Ghaymah monitoring — how to catch this before an outage using the platform's monitoring: per-container memory graphs (watch the sawtooth/creep pattern), alert rules (memory > 80% for 5 min, restarts > 3 in 10 min, OOMKilled events), dashboards and notification channels. Wrap any claim about a specific Ghaymah feature name in comments so I can check it against the real docs. + +Tone: professional SRE postmortem, markdown tables, no fluff. + +[GHAYMAH NOTES: ] +``` + +--- + +### PROMPT 3 → SOL (Q3: CI/CD) + +``` +Create two files for a CI/CD deliverable targeting the Ghaymah cloud platform: + +1. .github/workflows/deploy.yml — GitHub Actions workflow: + - Triggers: push to main, and workflow_dispatch + - Job build-push: checkout → docker/login-action to the Ghaymah container registry using secrets GHAYMAH_REGISTRY_USER and GHAYMAH_REGISTRY_TOKEN → docker/build-push-action building q1-deploy-monitor/app, tagging both //ghaymah-api:${{ github.sha }} and :latest + - Job deploy-staging (needs: build-push): installs the ghaymah CLI, authenticates with secret GHAYMAH_API_TOKEN, deploys/updates the STAGING service to the new image tag + - Job deploy-production (needs: deploy-staging): environment: production — this is the manual approval gate — then same deploy against the PRODUCTION service + - Mark every Ghaymah-specific command with a # VERIFY comment + +2. q3-cicd/CICD.md with: + - Pipeline overview + a mermaid diagram: commit → build+push → staging (auto) → manual approval → production + - "Manual approval" section: explain it's implemented via a GitHub Environment named production with required reviewers, configured in Settings → Environments (not in YAML), and the run pauses until approved + - "Staging vs Production" section: purpose of each, differences table (data: synthetic vs real; scale/replicas; secrets; access control; alerting thresholds; deploy cadence; who can approve) + - "Ghaymah CLI integration" section: install, auth login with API token, the deploy/update command, and how the workflow steps call it. Wrap uncertain command names in . + +[GHAYMAH NOTES: ] +``` + +--- + +### PROMPT 4 → SOL (Q4: scalability) + +``` +Write q4-scalability/SCALABILITY.md answering this task about the Ghaymah cloud platform: + +1. Architecture diagram (mermaid, renders on GitHub) for an app receiving 15,000 req/s: + DNS → CDN/edge cache → L7 load balancer → auto-scaled fleet of stateless API containers → Redis cache → PostgreSQL primary + read replicas (volumes on Ghaymah Block Storage) → and a monitoring/alerting component observing everything. Annotate the request flow and where bursts get absorbed (CDN + cache). + +2. Container count calculation — show the arithmetic explicitly: + 15,000 req/s ÷ 500 req/s per container = 30 containers + +30% headroom: 30 × 1.30 = 39 containers + Answer: 39. Add notes: always round up; keep N+1 extra for rolling deploys; headroom also covers traffic spikes and a zone failure. + +3. Cold-start strategy for new containers, as concrete bullets: minimum warm replica floor (never scale to zero for this tier), slim pre-pulled images (small base, few layers) for fast pulls, fast app boot (lazy-load non-critical work), readiness probe so the load balancer only routes to warmed containers, predictive/scheduled scale-up before known peaks, step-based scale-out (add several at once under sharp load). + +4. Ghaymah Block Storage for stateful workloads: containers are ephemeral so state must live on attached volumes; use Block Storage for the database, queues, uploads; one-writer-per-volume semantics mean the stateful tier scales via replication (primary/replica) not by cloning volumes; snapshots for backup; IOPS/size considerations; the API tier stays completely diskless so it can scale freely. Wrap platform-specific claims in . + +[GHAYMAH NOTES: ] +``` + +--- + +### PROMPT 5 → OPUS (Q5: mithal.space monitoring) + +``` +Build a monitoring system for the website https://mithal.space (a search engine), in q5-mithal-dashboard/: + +1. collector/collect.py — Python (stdlib + requests only): + Every 60 seconds (loop mode; also support --once) measure: + - latency_ms: timed GET https://mithal.space (10s timeout) + - up: true if status code 200–399 + - code: the status code (null on connection failure) + - ssl_days_left: open an ssl socket to mithal.space:443, read the cert notAfter, compute days remaining + - dns_ms: timed socket.getaddrinfo("mithal.space", 443) + - search_ms: timed GET to the search endpoint — make the URL a config constant, default: + Append one object per run to data/metrics.json (JSON array): + {"ts":"","up":bool,"code":int|null,"latency_ms":float|null,"dns_ms":float|null,"ssl_days_left":int|null,"search_ms":float|null} + Prune entries older than 48h on each write. Never crash on a failed check — record nulls. + +2. dashboard/index.html — ONE self-contained file (Chart.js CDN allowed) fetching data/metrics.json (path const at top): + - Uptime % tile over the last 24h: up_checks / total_checks × 100, one decimal + - Line chart of latency_ms for the LAST HOUR (optionally search_ms as second series) + - SSL card: "X days remaining", green if >30, yellow 8–30, red ≤7 + - Table of the last 10 checks: time, ✅/❌, code, latency, dns, search + - Auto-refresh every 60s, graceful empty state, clean professional look consistent with a dark ops-dashboard style + +3. Dockerfile (in q5-mithal-dashboard/) — a single container that BOTH serves the dashboard as static files on port 8080 AND runs the collector in the background writing into the served directory. Simplest approach: python:3.12-slim, a start.sh entrypoint that launches collect.py in the background then runs python -m http.server 8080 (or uvicorn static serving) from the dashboard directory, with data/ inside the served path so the dashboard can fetch metrics.json. + +4. README.md: run locally, build the image, deploy to Ghaymah. + +[GHAYMAH NOTES: ] +``` + +--- + +## PART C — Ready [GHAYMAH NOTES] blocks (paste these as-is) + +Research result: Ghaymah docs only document **deploy-from-image-URL** (image URL + app name + port + public access toggle + env vars → Deploy). No registry, CLI, monitoring, autoscaling, or block-storage details are documented. Strategy: use **Docker Hub** as the registry, and have agents label undocumented platform specifics as **"Proposed design"** instead of inventing features. mithal.space search URL confirmed: `https://mithal.space/search?q=`. + +### Paste into PROMPT 1 (Opus, Q1): +``` +[GHAYMAH NOTES] +- Ghaymah deploys containers from a public image URL. Deployment flow (dashboard): enter Container Image URL (e.g. docker.io//ghaymah-api:latest) → set Application Name → set Port Number (must match the EXPOSEd port) → enable Public Access → add Environment Variables → click Deploy. +- So in the README, document: build locally → docker push to Docker Hub (docker.io//ghaymah-api) → deploy on Ghaymah by pasting that image URL with port 8080 and Public Access enabled. +- Ghaymah's docs don't document the public URL format; write "" as a placeholder. +``` + +### Paste into PROMPT 2 (Sol, Q2): +``` +[GHAYMAH NOTES] +- Ghaymah's public docs do not document monitoring/alerting or autoscaling features. IMPORTANT: do NOT invent Ghaymah feature names. Instead: + - Section 5 (auto-scaling policy): the task literally asks to DESIGN a policy for the platform — present it as a proposed policy design ("Proposed auto-scaling policy for Ghaymah"), which is exactly what's asked. + - Section 6 (early detection): frame as "monitoring approach on Ghaymah" using platform-agnostic container signals (memory %, restart count, OOMKilled events, exit code 137) and note these can be collected via an external monitor hitting /health plus container runtime stats, since that's verifiable. Keep markers only where you reference a platform capability. +``` + +### Paste into PROMPT 3 (Sol, Q3): +``` +[GHAYMAH NOTES] +- Ghaymah deploys from a container image URL entered in its dashboard (image URL + app name + port + public access + env vars → Deploy). No Ghaymah-hosted registry or CLI is documented publicly. +- Therefore: build the workflow to push to Docker Hub (docker/login-action with secrets DOCKERHUB_USERNAME / DOCKERHUB_TOKEN, image docker.io//ghaymah-api:${{ github.sha }} and :latest). Add a note: "if a Ghaymah-hosted registry is available, only the login server and image prefix change." +- For the deploy jobs: since no CLI is documented, implement deploy-staging and deploy-production as jobs that (a) print the exact image tag to deploy and (b) call a placeholder script scripts/ghaymah_deploy.sh marked # VERIFY, and document in CICD.md that per current docs the deploy step is updating the image URL/tag in the Ghaymah dashboard for the app — with the CLI section written as "integration guide (to be confirmed against Ghaymah CLI docs)". +- Staging vs production on Ghaymah: two separate deployed apps, e.g. myapp-staging and myapp-production, each with its own env vars; the workflow deploys staging automatically and production only after the GitHub Environment approval. +``` + +### Paste into PROMPT 4 (Sol, Q4): +``` +[GHAYMAH NOTES] +- Ghaymah's public docs don't detail Block Storage or autoscaling specifics. Write section 4 as a correct general explanation of block storage for stateful container workloads applied to Ghaymah ("Ghaymah Block Storage"), and keep only on hard specifics (size limits, snapshot support). The architecture/math/cold-start sections are platform-agnostic — no changes needed. +``` + +### Paste into PROMPT 5 (Opus, Q5): +``` +[GHAYMAH NOTES] +- Search endpoint CONFIRMED working: https://mithal.space/search?q=test (returns a results page). Use that as the default search URL constant. +- Deployment: Ghaymah deploys from a public image URL (dashboard: image URL + app name + port 8080 + Public Access enabled + env vars → Deploy). README should say: push image to Docker Hub, then deploy by pasting docker.io//mithal-monitor:latest into Ghaymah with port 8080. +``` + +### Your remaining manual to-dos (updated) +1. Make/confirm a **Docker Hub** account; create repos `ghaymah-api` and `mithal-monitor` (public is simplest for Ghaymah to pull). +2. In GitHub: secrets `DOCKERHUB_USERNAME`, `DOCKERHUB_TOKEN` (access token from Docker Hub → Account Settings → Security); environment `production` with you as required reviewer. +3. In the Ghaymah dashboard, while deploying, **look for tabs/menus the docs didn't cover** (Registry, CLI, Metrics/Logs, Scaling, Volumes). If any exist, screenshot them and resolve the matching markers — instant answer upgrades for Q2/Q3/Q4. +4. Screenshots needed: Q1 app running + /health JSON, Q3 workflow paused at approval, Q5 live dashboard. diff --git a/README.md b/README.md new file mode 100644 index 0000000..9d2e6fa --- /dev/null +++ b/README.md @@ -0,0 +1,83 @@ +# Ghaymah Cloud Technical Assessment + +This repository contains the five deliverables for the Ghaymah cloud platform assessment. + +## Deliverables + +| Question | Deliverable | Status | +|---|---|---| +| Q1 — Deploy and monitor an API | [Application, monitor, dashboard, and deployment guide](q1-deploy-monitor/README.md) | Implementation complete; live Ghaymah deployment evidence pending | +| Q2 — OOMKilled postmortem | [POSTMORTEM.md](q2-postmortem/POSTMORTEM.md) | Complete | +| Q3 — CI/CD pipeline | [Workflow](.github/workflows/deploy.yml) and [CICD.md](q3-cicd/CICD.md) | Build/push and approval gate complete; automated Ghaymah image update awaiting a documented non-interactive command | +| Q4 — Scalability | [SCALABILITY.md](q4-scalability/SCALABILITY.md) | Complete | +| Q5 — mithal.space monitoring | [Collector and dashboard](q5-mithal-dashboard/) | Implementation present; live deployment evidence pending | + +## Current verified Ghaymah information + +The current public Ghaymah documentation establishes the following: + +- The official CLI is installed with `curl -sSL https://cli.ghaymah.systems/install.sh | bash`. +- The executable is named `gy`; the current binary reports its release with `gy version`. +- Authentication uses `gy auth login`. The current CLI exposes `--email` and `--password` flags, but does not expose an API-token flag. +- `gy resource app init --project-id --name ` creates `.ghaymah.json`. +- `gy resource app launch [PATH]` builds and deploys the application described by the local Dockerfile and `.ghaymah.json`. +- `gy resource app logs` retrieves application logs. +- `gy resource app update ` accepts JSON input or dot-notation updates, but the public help does not identify a supported external-image field. +- The documented `.ghaymah.json` example includes the application ID/name, project ID, exposed port, public access, resource tier, and Dockerfile name. +- Ghaymah lists a container registry, CI/CD, monitoring, autoscaling, APIs, and Block Storage as products or capabilities, but the public pages reviewed for this submission do not expose enough operational syntax or limits to use their hard specifics safely. +- The authenticated deployment form supports either a Git repository or container image URL, an optional registry pull secret, instance size, application name, port, public access, custom domain, environment variables, and attached storage volumes. +- The authenticated volume form displays a supported size range of 50 MiB to 10 GiB. +- The authenticated External Integrations page exposes Docker Hub connection fields; no Ghaymah-hosted registry endpoint or push instructions were visible. + +Official references: + +- [Ghaymah CLI documentation](https://ghaymah.systems/docs) +- [Ghaymah CLI overview](https://ghaymah.systems/cli) +- [Ghaymah products](https://ghaymah.systems/products) +- [Ghaymah changelog](https://ghaymah.systems/changelog) + +## Local quick checks + +### Q1 API + +```bash +python -m pip install -r q1-deploy-monitor/app/requirements.txt +python -m uvicorn main:app --app-dir q1-deploy-monitor/app --host 0.0.0.0 --port 8080 +``` + +Then open: + +- `http://localhost:8080/health` +- `http://localhost:8080/metrics` +- `http://localhost:8080/docs` + +### Q1 monitor and dashboard + +```bash +APP_URL=http://localhost:8080 python q1-deploy-monitor/monitor/monitor.py --once +python -m http.server 8000 --directory q1-deploy-monitor +``` + +Open `http://localhost:8000/dashboard/index.html`. + +### Q5 collector + +```bash +python -m pip install -r q5-mithal-dashboard/collector/requirements.txt +python q5-mithal-dashboard/collector/collect.py --once +``` + +## Work that requires account access + +The following cannot be completed safely from source code alone: + +1. Create the Ghaymah projects/applications and obtain their IDs and public URLs. +2. Push the Docker images using the owner's registry credentials. +3. Deploy Q1 and Q5 and capture the required screenshots. +4. Configure GitHub secrets and the `production` Environment with required reviewers. +5. Confirm the supported non-interactive Ghaymah authentication mechanism for GitHub Actions. +6. Confirm the command/API that updates an existing Ghaymah application to a specific external image tag. +7. Run the workflow, approve production, and capture the approval evidence. +8. Confirm monitoring/autoscaling controls and Block Storage limits in the authenticated dashboard before removing remaining `VERIFY` markers. + +No credentials should be committed to this repository. Use GitHub Secrets and Ghaymah's secret-management controls for sensitive values. diff --git a/q1-deploy-monitor/README.md b/q1-deploy-monitor/README.md new file mode 100644 index 0000000..0b52250 --- /dev/null +++ b/q1-deploy-monitor/README.md @@ -0,0 +1,262 @@ +# Q1 — Deploy & Monitor an API on Ghaymah + +A minimal FastAPI service, containerised and deployed on the **Ghaymah** container +platform, plus a polling monitor and a static dashboard that visualises its +availability, response time and request count. + +``` +q1-deploy-monitor/ +├── app/ +│ ├── main.py # FastAPI app: /, /health, /metrics +│ ├── requirements.txt # pinned fastapi + uvicorn +│ ├── Dockerfile # python:3.12-slim, non-root, HEALTHCHECK +│ └── .dockerignore +├── monitor/ +│ ├── monitor.py # polls /health + /metrics every 30s (stdlib only) +│ └── data/checks.json # created on first run — the dashboard's data source +├── dashboard/ +│ └── index.html # single self-contained page (Chart.js from CDN) +└── README.md +``` + +--- + +## 1. The API + +| Method | Path | Response | +|---|---|---| +| `GET` | `/` | service name, version, endpoint list, start time | +| `GET` | `/health` | `{"status":"ok","uptime_s":12.34,"timestamp":"2026-07-26T15:36:50.283265+00:00"}` — HTTP 200 | +| `GET` | `/metrics` | `{"requests_total":42,"started_at":""}` | +| `GET` | `/docs` | interactive OpenAPI docs (FastAPI built-in) | + +`/health` performs **no** downstream checks (no DB, no network) — it reports +process liveness only, so a failing check always means "restart me", which is +exactly the signal an orchestrator's health probe should act on. + +`/metrics` is backed by an in-memory counter incremented by an HTTP middleware on +every request. It is per-process and deliberately resets on restart — a counter +that drops to zero in the dashboard is a visible signal that the container was +restarted or redeployed. + +Environment variables (all optional): `APP_NAME`, `APP_VERSION`, `PORT` (default `8080`). + +### Run locally without Docker + +```bash +cd q1-deploy-monitor/app +python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r requirements.txt +python main.py # or: uvicorn main:app --host 0.0.0.0 --port 8080 +``` + +```bash +curl http://localhost:8080/health +``` + +--- + +## 2. Build and run the Docker image locally + +```bash +docker build -t ghaymah-api:latest ./q1-deploy-monitor/app +``` + +```bash +docker run --rm -p 8080:8080 --name ghaymah-api ghaymah-api:latest +``` + +Verify: + +```bash +curl -f http://localhost:8080/health && curl http://localhost:8080/metrics +``` + +Docker's own health probe (defined by `HEALTHCHECK` in the Dockerfile) shows up +after ~10s in `docker ps` as `(healthy)`: + +```bash +docker ps --filter name=ghaymah-api +``` + +**Image design notes** (what the Dockerfile is doing and why): + +- `python:3.12-slim` — small base, no build toolchain in the final image. +- `requirements.txt` is copied and installed **before** the application code, so + editing `main.py` reuses the cached dependency layer instead of reinstalling + FastAPI on every build. +- Runs as the non-root user `appuser` (uid 10001). +- `curl` is the only extra apt package, installed solely for the `HEALTHCHECK`; + apt lists are removed in the same layer. +- `EXPOSE 8080` matches the port Ghaymah is configured with below. + +--- + +## 3. Push the image and deploy on Ghaymah + +Ghaymah deploys a container from a **public image URL**, so the image must live in +a public registry first. Docker Hub is used here — replace `` +with your own account name. + +### 3.1 Push to Docker Hub + +```bash +docker login +``` + +```bash +docker tag ghaymah-api:latest docker.io//ghaymah-api:latest +``` + +```bash +docker push docker.io//ghaymah-api:latest +``` + +> Building on an Apple Silicon / ARM machine? Build for the platform Ghaymah runs +> (`linux/amd64`) or the container will fail to start: +> ```bash +> docker buildx build --platform linux/amd64 -t docker.io//ghaymah-api:latest --push ./q1-deploy-monitor/app +> ``` + +Make sure the Docker Hub repository is **public** — Ghaymah pulls the image +anonymously from the URL you paste in. + +### 3.2 Deploy on the Ghaymah dashboard + +| Field | Value | +|---|---| +| Container Image URL | `docker.io//ghaymah-api:latest` | +| Application Name | `ghaymah-api` | +| Port Number | `8080` (must match the `EXPOSE`d port) | +| Public Access | **enabled** | +| Environment Variables | *(optional)* `APP_NAME=ghaymah-api`, `APP_VERSION=1.0.0` | + +Then click **Deploy**. Once the deployment reports as running, Ghaymah assigns the +service a public URL. + +### 3.3 Verify the live deployment + +```bash +curl -f /health +``` + +Expected: HTTP 200 with `{"status":"ok","uptime_s":...,"timestamp":"..."}`. + +Also open `/docs` in a browser for the OpenAPI page, +and screenshot both the running service in the Ghaymah dashboard and the `/health` +response for the submission. + +> **Redeploying a new version:** push a new image tag and update the Container +> Image URL on the service. Prefer an explicit tag (e.g. `:v2` or the git SHA) +> over `:latest` so a redeploy is unambiguous about which build is running. + +--- + +## 4. Run the monitor + +`monitor/monitor.py` uses the **Python standard library only** — nothing to install. + +```bash +export APP_URL="" +python q1-deploy-monitor/monitor/monitor.py +``` + +On Windows PowerShell: + +```bash +$env:APP_URL=""; python q1-deploy-monitor\monitor\monitor.py +``` + +Every 30 seconds it issues `GET $APP_URL/health` with a 5 s timeout, then +`GET $APP_URL/metrics`, and appends one record to `monitor/data/checks.json` +(a JSON array, created on first run): + +```json +{ + "ts": "2026-07-26T15:37:29.886772+00:00", + "status": "up", + "code": 200, + "latency_ms": 65.3, + "requests": 2 +} +``` + +- Any timeout, connection error, TLS failure or non-2xx response → `"status":"down"` + with `latency_ms: null` (and the HTTP code when the server did answer). +- `/metrics` is best-effort: if only that call fails, the check still counts as + **up** and `requests` is `null`. +- After **3 consecutive failures** it prints an `ALERT:` line to stdout (once per + outage), and a `RECOVERED:` line when the service answers again. + +Single check (useful for cron, CI or a smoke test — exits `0` if up, `1` if down): + +```bash +APP_URL="" python q1-deploy-monitor/monitor/monitor.py --once +``` + +Leave the loop running well before the submission so the dashboard has real history. + +**Tuning via environment variables** + +| Variable | Default | Meaning | +|---|---|---| +| `APP_URL` | *(required)* | base URL of the deployed app (also settable with `--url`) | +| `INTERVAL_S` | `30` | seconds between checks | +| `TIMEOUT_S` | `5` | per-request timeout | +| `ALERT_AFTER` | `3` | consecutive failures before the ALERT line | +| `DATA_FILE` | `monitor/data/checks.json` | where records are written | +| `MAX_RECORDS` | `2880` | rolling window (24 h at one check / 30 s) | + +Records are written atomically (temp file + rename), so the dashboard never reads +a half-written file. + +--- + +## 5. Open the dashboard + +The page fetches `../monitor/data/checks.json`, so it must be served over HTTP — +opening `index.html` directly from the filesystem is blocked by the browser's +`file://` fetch restrictions (the page detects this and tells you so instead of +failing silently). + +```bash +cd q1-deploy-monitor && python -m http.server 8000 +``` + +Then open . + +It shows: + +- **Status badge** — green `UP` / red `DOWN` from the most recent check, with the + HTTP code and timestamp. +- **Total requests** — `requests_total` from the latest check. +- **Latest latency** + the average across the stored window. +- **Uptime %** across all stored checks. +- **Latency line chart** — `latency_ms` over time (last 120 checks); down checks + appear as gaps with red points. +- **Last-updated timestamp**, auto-refreshing every 30 s. + +With no data (monitor not started yet, file missing, or an empty/corrupt array) +it renders a "no data yet" state and an explanatory banner rather than erroring. + +To point the page at a different data file, edit the one constant at the top of +the ` + + + +
+ +
+

Ghaymah API · monitoring

+
Last updated —
+
+ + + +
+
+
Current status
+
NO DATA
+
Waiting for the first check…
+
+ +
+
Total requests
+
—
+
from /metrics
+
+ +
+
Latest latency
+
—
+
avg — · checks —
+
+ +
+
Uptime (all checks)
+
—
+
— up / — down
+
+
+ +
+
+
Response time — latency_ms over time
+
+
+
+ + +
+
+ +
+ Data source: + Auto-refresh every 30s +
+
+ + + + diff --git a/q1-deploy-monitor/monitor/data/checks.json b/q1-deploy-monitor/monitor/data/checks.json new file mode 100644 index 0000000..d60a3b7 --- /dev/null +++ b/q1-deploy-monitor/monitor/data/checks.json @@ -0,0 +1,625 @@ +[ + { + "ts": "2026-07-26T16:00:54.022563+00:00", + "status": "up", + "code": 200, + "latency_ms": 48.44, + "requests": 5 + }, + { + "ts": "2026-07-26T16:01:24.088718+00:00", + "status": "up", + "code": 200, + "latency_ms": 12.02, + "requests": 8 + }, + { + "ts": "2026-07-26T16:01:54.237407+00:00", + "status": "up", + "code": 200, + "latency_ms": 24.42, + "requests": 11 + }, + { + "ts": "2026-07-26T16:02:24.326513+00:00", + "status": "up", + "code": 200, + "latency_ms": 10.23, + "requests": 14 + }, + { + "ts": "2026-07-26T16:02:54.359082+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.32, + "requests": 19 + }, + { + "ts": "2026-07-26T16:03:24.389873+00:00", + "status": "up", + "code": 200, + "latency_ms": 7.83, + "requests": 22 + }, + { + "ts": "2026-07-26T16:03:54.430952+00:00", + "status": "up", + "code": 200, + "latency_ms": 11.57, + "requests": 25 + }, + { + "ts": "2026-07-26T16:04:24.463587+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.01, + "requests": 28 + }, + { + "ts": "2026-07-26T16:04:54.502315+00:00", + "status": "up", + "code": 200, + "latency_ms": 11.91, + "requests": 31 + }, + { + "ts": "2026-07-26T16:05:24.545377+00:00", + "status": "up", + "code": 200, + "latency_ms": 7.72, + "requests": 34 + }, + { + "ts": "2026-07-26T16:05:54.596001+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.86, + "requests": 37 + }, + { + "ts": "2026-07-26T16:06:24.623029+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.53, + "requests": 40 + }, + { + "ts": "2026-07-26T16:06:54.659187+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.9, + "requests": 43 + }, + { + "ts": "2026-07-26T16:07:24.697242+00:00", + "status": "up", + "code": 200, + "latency_ms": 11.24, + "requests": 46 + }, + { + "ts": "2026-07-26T16:07:54.732633+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.15, + "requests": 49 + }, + { + "ts": "2026-07-26T16:08:24.764311+00:00", + "status": "up", + "code": 200, + "latency_ms": 7.73, + "requests": 52 + }, + { + "ts": "2026-07-26T16:08:54.794053+00:00", + "status": "up", + "code": 200, + "latency_ms": 7.53, + "requests": 55 + }, + { + "ts": "2026-07-26T16:09:24.830576+00:00", + "status": "up", + "code": 200, + "latency_ms": 10.88, + "requests": 58 + }, + { + "ts": "2026-07-26T16:09:55.392240+00:00", + "status": "up", + "code": 200, + "latency_ms": 9.02, + "requests": 61 + }, + { + "ts": "2026-07-26T16:10:25.432620+00:00", + "status": "up", + "code": 200, + "latency_ms": 7.75, + "requests": 64 + }, + { + "ts": "2026-07-26T16:10:55.469947+00:00", + "status": "up", + "code": 200, + "latency_ms": 9.34, + "requests": 67 + }, + { + "ts": "2026-07-26T16:11:25.502913+00:00", + "status": "up", + "code": 200, + "latency_ms": 7.92, + "requests": 70 + }, + { + "ts": "2026-07-26T16:11:55.542608+00:00", + "status": "up", + "code": 200, + "latency_ms": 11.26, + "requests": 73 + }, + { + "ts": "2026-07-26T16:12:25.579631+00:00", + "status": "up", + "code": 200, + "latency_ms": 10.91, + "requests": 76 + }, + { + "ts": "2026-07-26T16:12:55.621565+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.02, + "requests": 79 + }, + { + "ts": "2026-07-26T16:13:25.651738+00:00", + "status": "up", + "code": 200, + "latency_ms": 7.58, + "requests": 82 + }, + { + "ts": "2026-07-26T16:13:55.683853+00:00", + "status": "up", + "code": 200, + "latency_ms": 9.19, + "requests": 85 + }, + { + "ts": "2026-07-26T16:14:25.720913+00:00", + "status": "up", + "code": 200, + "latency_ms": 10.67, + "requests": 88 + }, + { + "ts": "2026-07-26T16:14:55.754438+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.68, + "requests": 91 + }, + { + "ts": "2026-07-26T16:15:25.795149+00:00", + "status": "up", + "code": 200, + "latency_ms": 9.94, + "requests": 94 + }, + { + "ts": "2026-07-26T16:15:55.835606+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.67, + "requests": 97 + }, + { + "ts": "2026-07-26T16:16:25.894224+00:00", + "status": "up", + "code": 200, + "latency_ms": 21.72, + "requests": 100 + }, + { + "ts": "2026-07-26T16:16:55.947740+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.81, + "requests": 103 + }, + { + "ts": "2026-07-26T16:17:25.993670+00:00", + "status": "up", + "code": 200, + "latency_ms": 14.76, + "requests": 106 + }, + { + "ts": "2026-07-26T16:17:56.027955+00:00", + "status": "up", + "code": 200, + "latency_ms": 9.95, + "requests": 109 + }, + { + "ts": "2026-07-26T16:18:26.076150+00:00", + "status": "up", + "code": 200, + "latency_ms": 16.53, + "requests": 112 + }, + { + "ts": "2026-07-26T16:18:56.130454+00:00", + "status": "up", + "code": 200, + "latency_ms": 13.91, + "requests": 115 + }, + { + "ts": "2026-07-26T16:19:26.181118+00:00", + "status": "up", + "code": 200, + "latency_ms": 14.55, + "requests": 118 + }, + { + "ts": "2026-07-26T16:19:56.239466+00:00", + "status": "up", + "code": 200, + "latency_ms": 15.54, + "requests": 121 + }, + { + "ts": "2026-07-26T16:20:26.273327+00:00", + "status": "up", + "code": 200, + "latency_ms": 12.98, + "requests": 124 + }, + { + "ts": "2026-07-26T16:20:56.326056+00:00", + "status": "up", + "code": 200, + "latency_ms": 10.54, + "requests": 127 + }, + { + "ts": "2026-07-26T16:21:26.360754+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.28, + "requests": 130 + }, + { + "ts": "2026-07-26T16:21:56.419699+00:00", + "status": "up", + "code": 200, + "latency_ms": 24.33, + "requests": 133 + }, + { + "ts": "2026-07-26T16:22:26.474809+00:00", + "status": "up", + "code": 200, + "latency_ms": 16.0, + "requests": 136 + }, + { + "ts": "2026-07-26T16:22:56.537278+00:00", + "status": "up", + "code": 200, + "latency_ms": 25.59, + "requests": 139 + }, + { + "ts": "2026-07-26T16:23:26.588750+00:00", + "status": "up", + "code": 200, + "latency_ms": 11.74, + "requests": 142 + }, + { + "ts": "2026-07-26T16:23:56.625763+00:00", + "status": "up", + "code": 200, + "latency_ms": 11.76, + "requests": 145 + }, + { + "ts": "2026-07-26T16:24:26.678666+00:00", + "status": "up", + "code": 200, + "latency_ms": 21.03, + "requests": 148 + }, + { + "ts": "2026-07-26T16:24:56.718058+00:00", + "status": "up", + "code": 200, + "latency_ms": 9.81, + "requests": 151 + }, + { + "ts": "2026-07-26T16:25:26.781135+00:00", + "status": "up", + "code": 200, + "latency_ms": 26.99, + "requests": 154 + }, + { + "ts": "2026-07-26T16:25:56.837517+00:00", + "status": "up", + "code": 200, + "latency_ms": 17.22, + "requests": 157 + }, + { + "ts": "2026-07-26T16:26:26.885310+00:00", + "status": "up", + "code": 200, + "latency_ms": 15.99, + "requests": 160 + }, + { + "ts": "2026-07-26T16:26:56.931050+00:00", + "status": "up", + "code": 200, + "latency_ms": 12.71, + "requests": 163 + }, + { + "ts": "2026-07-26T16:27:26.978657+00:00", + "status": "up", + "code": 200, + "latency_ms": 15.71, + "requests": 166 + }, + { + "ts": "2026-07-26T16:27:57.027222+00:00", + "status": "up", + "code": 200, + "latency_ms": 15.87, + "requests": 169 + }, + { + "ts": "2026-07-26T16:28:27.076324+00:00", + "status": "up", + "code": 200, + "latency_ms": 11.98, + "requests": 172 + }, + { + "ts": "2026-07-26T16:28:57.144076+00:00", + "status": "up", + "code": 200, + "latency_ms": 24.72, + "requests": 175 + }, + { + "ts": "2026-07-26T16:29:27.203425+00:00", + "status": "up", + "code": 200, + "latency_ms": 20.56, + "requests": 178 + }, + { + "ts": "2026-07-26T16:29:57.243755+00:00", + "status": "up", + "code": 200, + "latency_ms": 7.91, + "requests": 181 + }, + { + "ts": "2026-07-26T16:30:27.305659+00:00", + "status": "up", + "code": 200, + "latency_ms": 19.38, + "requests": 184 + }, + { + "ts": "2026-07-26T16:30:57.380343+00:00", + "status": "up", + "code": 200, + "latency_ms": 37.99, + "requests": 187 + }, + { + "ts": "2026-07-26T16:31:27.412862+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.02, + "requests": 190 + }, + { + "ts": "2026-07-26T16:31:57.458486+00:00", + "status": "up", + "code": 200, + "latency_ms": 16.76, + "requests": 193 + }, + { + "ts": "2026-07-26T16:32:27.506190+00:00", + "status": "up", + "code": 200, + "latency_ms": 10.54, + "requests": 196 + }, + { + "ts": "2026-07-26T16:32:57.558316+00:00", + "status": "up", + "code": 200, + "latency_ms": 19.22, + "requests": 199 + }, + { + "ts": "2026-07-26T16:33:27.636370+00:00", + "status": "up", + "code": 200, + "latency_ms": 18.47, + "requests": 202 + }, + { + "ts": "2026-07-26T16:33:57.692190+00:00", + "status": "up", + "code": 200, + "latency_ms": 20.24, + "requests": 205 + }, + { + "ts": "2026-07-26T16:34:27.737228+00:00", + "status": "up", + "code": 200, + "latency_ms": 10.93, + "requests": 208 + }, + { + "ts": "2026-07-26T16:34:57.800304+00:00", + "status": "up", + "code": 200, + "latency_ms": 22.88, + "requests": 211 + }, + { + "ts": "2026-07-26T16:35:27.861991+00:00", + "status": "up", + "code": 200, + "latency_ms": 18.19, + "requests": 214 + }, + { + "ts": "2026-07-26T16:35:57.925683+00:00", + "status": "up", + "code": 200, + "latency_ms": 19.93, + "requests": 217 + }, + { + "ts": "2026-07-26T16:36:27.992496+00:00", + "status": "up", + "code": 200, + "latency_ms": 18.07, + "requests": 220 + }, + { + "ts": "2026-07-26T16:36:58.067518+00:00", + "status": "up", + "code": 200, + "latency_ms": 18.62, + "requests": 223 + }, + { + "ts": "2026-07-26T16:37:28.127809+00:00", + "status": "up", + "code": 200, + "latency_ms": 20.59, + "requests": 226 + }, + { + "ts": "2026-07-26T16:37:58.166210+00:00", + "status": "up", + "code": 200, + "latency_ms": 9.57, + "requests": 229 + }, + { + "ts": "2026-07-26T16:38:28.201786+00:00", + "status": "up", + "code": 200, + "latency_ms": 7.95, + "requests": 232 + }, + { + "ts": "2026-07-26T16:38:58.237767+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.12, + "requests": 235 + }, + { + "ts": "2026-07-26T16:39:28.279617+00:00", + "status": "up", + "code": 200, + "latency_ms": 11.27, + "requests": 238 + }, + { + "ts": "2026-07-26T16:39:58.317037+00:00", + "status": "up", + "code": 200, + "latency_ms": 7.61, + "requests": 241 + }, + { + "ts": "2026-07-26T16:40:28.355043+00:00", + "status": "up", + "code": 200, + "latency_ms": 9.57, + "requests": 244 + }, + { + "ts": "2026-07-26T16:40:58.391288+00:00", + "status": "up", + "code": 200, + "latency_ms": 9.27, + "requests": 247 + }, + { + "ts": "2026-07-26T16:41:28.432715+00:00", + "status": "up", + "code": 200, + "latency_ms": 13.3, + "requests": 250 + }, + { + "ts": "2026-07-26T16:41:58.503792+00:00", + "status": "up", + "code": 200, + "latency_ms": 26.98, + "requests": 253 + }, + { + "ts": "2026-07-26T16:42:28.608440+00:00", + "status": "up", + "code": 200, + "latency_ms": 19.86, + "requests": 256 + }, + { + "ts": "2026-07-26T16:42:58.693278+00:00", + "status": "up", + "code": 200, + "latency_ms": 12.8, + "requests": 259 + }, + { + "ts": "2026-07-26T16:43:28.797562+00:00", + "status": "up", + "code": 200, + "latency_ms": 42.28, + "requests": 262 + }, + { + "ts": "2026-07-26T16:43:58.874830+00:00", + "status": "up", + "code": 200, + "latency_ms": 8.97, + "requests": 265 + }, + { + "ts": "2026-07-26T16:44:28.919903+00:00", + "status": "up", + "code": 200, + "latency_ms": 13.56, + "requests": 268 + }, + { + "ts": "2026-07-26T16:44:59.037760+00:00", + "status": "up", + "code": 200, + "latency_ms": 34.65, + "requests": 271 + } +] diff --git a/q1-deploy-monitor/monitor/monitor.py b/q1-deploy-monitor/monitor/monitor.py new file mode 100644 index 0000000..a653c74 --- /dev/null +++ b/q1-deploy-monitor/monitor/monitor.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Uptime monitor for the API deployed on Ghaymah. + +Polls ``$APP_URL/health`` (and ``/metrics``) every 30 seconds and appends one +record per check to ``monitor/data/checks.json`` — the file the dashboard reads. + +Record schema: + {"ts": "", "status": "up"|"down", "code": , + "latency_ms": , "requests": } + +Usage: + APP_URL=https://my-app.example python monitor.py + APP_URL=https://my-app.example python monitor.py --once + +Standard library only — no pip install needed to run the monitor. +""" + +import argparse +import json +import os +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +# --- Configuration (env-overridable) ----------------------------------------- + +APP_URL = os.getenv("APP_URL", "").rstrip("/") +INTERVAL_S = float(os.getenv("INTERVAL_S", "30")) +TIMEOUT_S = float(os.getenv("TIMEOUT_S", "5")) +ALERT_AFTER = int(os.getenv("ALERT_AFTER", "3")) + +DEFAULT_DATA_FILE = Path(__file__).resolve().parent / "data" / "checks.json" +DATA_FILE = Path(os.getenv("DATA_FILE", str(DEFAULT_DATA_FILE))) + +# Keep the JSON array bounded so the dashboard stays fast and the file small. +MAX_RECORDS = int(os.getenv("MAX_RECORDS", "2880")) # 24h at one check / 30s + + +def now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def fetch_json(url: str, timeout: float): + """GET a URL and parse JSON. Returns (status_code, parsed_body). + + Raises on timeout, connection failure, or non-2xx status. + """ + req = urllib.request.Request(url, headers={"User-Agent": "ghaymah-monitor/1.0"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8", errors="replace") + try: + body = json.loads(raw) + except json.JSONDecodeError: + body = None + return resp.status, body + + +def check_once(app_url: str) -> dict: + """Run one health + metrics check and return the record to persist.""" + health_url = f"{app_url}/health" + metrics_url = f"{app_url}/metrics" + + started = time.perf_counter() + try: + code, _body = fetch_json(health_url, TIMEOUT_S) + latency_ms = round((time.perf_counter() - started) * 1000, 2) + status = "up" + except urllib.error.HTTPError as exc: + # The app answered, just not with a healthy status — record the code. + return { + "ts": now_iso(), + "status": "down", + "code": exc.code, + "latency_ms": round((time.perf_counter() - started) * 1000, 2), + "requests": None, + } + except Exception: + # Timeout, DNS failure, connection refused, TLS error, ... + return { + "ts": now_iso(), + "status": "down", + "code": None, + "latency_ms": None, + "requests": None, + } + + # /metrics is best-effort: a failure there must not turn a healthy app "down". + requests_total = None + try: + _mcode, mbody = fetch_json(metrics_url, TIMEOUT_S) + if isinstance(mbody, dict): + value = mbody.get("requests_total") + if isinstance(value, int): + requests_total = value + except Exception: + pass + + return { + "ts": now_iso(), + "status": status, + "code": code, + "latency_ms": latency_ms, + "requests": requests_total, + } + + +def load_records(path: Path) -> list: + """Read the existing JSON array; tolerate a missing or corrupt file.""" + if not path.exists(): + return [] + try: + with path.open("r", encoding="utf-8") as fh: + data = json.load(fh) + return data if isinstance(data, list) else [] + except (json.JSONDecodeError, OSError): + print(f"[warn] {path} unreadable or corrupt — starting a fresh array", file=sys.stderr) + return [] + + +def append_record(path: Path, record: dict) -> None: + """Append one record to the JSON array, writing atomically.""" + path.parent.mkdir(parents=True, exist_ok=True) + records = load_records(path) + records.append(record) + if len(records) > MAX_RECORDS: + records = records[-MAX_RECORDS:] + + tmp = path.with_suffix(path.suffix + ".tmp") + with tmp.open("w", encoding="utf-8") as fh: + json.dump(records, fh, indent=2) + fh.write("\n") + tmp.replace(path) + + +def format_line(record: dict) -> str: + latency = f"{record['latency_ms']:.1f}ms" if record["latency_ms"] is not None else " - " + code = record["code"] if record["code"] is not None else "---" + reqs = record["requests"] if record["requests"] is not None else "-" + return ( + f"{record['ts']} {record['status'].upper():<4} " + f"code={code:<4} latency={latency:<9} requests={reqs}" + ) + + +def run(app_url: str, once: bool) -> int: + consecutive_failures = 0 + alerted = False + + while True: + record = check_once(app_url) + append_record(DATA_FILE, record) + print(format_line(record), flush=True) + + if record["status"] == "down": + consecutive_failures += 1 + if consecutive_failures >= ALERT_AFTER and not alerted: + print( + f"ALERT: {app_url} has failed {consecutive_failures} consecutive " + f"health checks (since {record['ts']})", + flush=True, + ) + alerted = True + else: + if alerted: + print(f"RECOVERED: {app_url} is responding again at {record['ts']}", flush=True) + consecutive_failures = 0 + alerted = False + + if once: + return 0 if record["status"] == "up" else 1 + + time.sleep(INTERVAL_S) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Monitor the Ghaymah-deployed API.") + parser.add_argument("--once", action="store_true", help="run a single check and exit") + parser.add_argument("--url", default=APP_URL, help="app base URL (default: $APP_URL)") + args = parser.parse_args() + + app_url = (args.url or "").rstrip("/") + if not app_url: + print( + "error: no app URL. Set APP_URL, e.g.\n" + " APP_URL=https://my-app.ghaymah.example python monitor.py", + file=sys.stderr, + ) + return 2 + + print(f"Monitoring {app_url} every {INTERVAL_S:g}s (timeout {TIMEOUT_S:g}s)", flush=True) + print(f"Writing checks to {DATA_FILE}", flush=True) + + try: + return run(app_url, args.once) + except KeyboardInterrupt: + print("\nStopped.", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/q2-postmortem/POSTMORTEM.md b/q2-postmortem/POSTMORTEM.md new file mode 100644 index 0000000..862faf7 --- /dev/null +++ b/q2-postmortem/POSTMORTEM.md @@ -0,0 +1,134 @@ +# Postmortem: Repeated OOMKilled Container Restarts + +**Severity:** SEV-2 +**Incident date:** 2026-07-24 +**Duration:** 45 minutes (14:05–14:50 EEST) +**Status:** Resolved + +## 1. Summary + +On 2026-07-24, the application was unavailable for 45 minutes after its containers entered a repeated `OOMKilled` crash loop with exit code 137. Memory usage had grown gradually following a release that introduced an unbounded in-process cache; once each container reached its configured memory limit, the container runtime terminated it. Automated client retries and queued traffic increased load on the briefly recovering containers, accelerating subsequent failures. Users experienced an estimated 95–100% request error rate during the incident, amounting to effective full downtime. The incident was classified as **SEV-2** because the production service was unavailable, but no data loss or security impact was identified. + +## 2. Timeline + +All times are EEST (UTC+3). + +| Time | Event | Actor/action | +|---|---|---| +| 13:40 | Release `v2.18.0` is deployed. It contains a new in-process response cache. | Deployment pipeline completes successfully; smoke checks pass. | +| 13:45 | Memory begins rising above the previous baseline of approximately 55% per container. | Application traffic populates the cache; no alert exists for sustained memory growth. | +| 13:52 | Memory reaches approximately 75% and continues to grow without returning to baseline. | No action is taken because memory-usage alerting is not configured. | +| 14:03 | One container approaches its configured memory limit; garbage collection activity and latency increase. | Container continues serving requests with degraded response times. | +| 14:05 | First container is terminated by the runtime as `OOMKilled` with exit code 137. | Runtime restart policy starts a replacement container; incident begins (minute 0). | +| 14:07 | The replacement becomes ready, receives queued and retried requests, and its cache grows rapidly. | Client retries and redistributed traffic increase pressure on the remaining containers. | +| 14:10 | Multiple containers enter a restart loop; error rate exceeds 95%. | Runtime repeatedly restarts terminated containers, but they fail again after becoming ready. | +| 14:13 | Availability alert fires after sustained health-check failures. | On-call engineer is paged and acknowledges the alert. | +| 14:16 | On-call confirms widespread HTTP 5xx/timeouts and repeated container restarts. | Engineer begins incident triage and declares SEV-2. | +| 14:20 | Exit code 137 and `OOMKilled` events are identified; memory graphs show steady growth after the release. | Engineer correlates the failure pattern with deployment `v2.18.0`. | +| 14:24 | Incident lead pauses nonessential changes and selects rollback as the primary mitigation. | A second engineer reviews the previous stable image and rollback procedure. | +| 14:28 | Memory limits are temporarily raised to provide recovery headroom while rollback proceeds. | Platform operator updates the workload configuration and replaces affected containers. | +| 14:33 | Rollback to `v2.17.4` starts. | Deployment pipeline replaces the leaking release with the last known-good image. | +| 14:38 | First rolled-back containers become healthy; error rate begins to fall. | On-call watches health checks, memory, restarts, latency, and request errors. | +| 14:42 | All active containers run `v2.17.4`; memory stabilizes near the prior baseline. | Traffic is allowed to normalize; retry pressure subsides. | +| 14:47 | Health checks remain successful and no new OOM kills occur for five minutes. | Incident lead begins the recovery validation period. | +| 14:50 | Error rate and latency return to normal; full service is confirmed. | Incident lead resolves the incident after 45 minutes of effective downtime. | + +## 3. Root Cause + +### 5-Whys analysis + +1. **Why was the application unavailable?** + Its production containers repeatedly terminated and restarted, leaving insufficient healthy capacity to serve requests. + +2. **Why did the containers terminate?** + Each container exceeded its memory limit and was killed by the container runtime, producing an `OOMKilled` event and exit code 137. + +3. **Why did memory exceed the limit?** + Release `v2.18.0` introduced an unbounded in-process response cache. Cache entries were not evicted, so memory grew continuously with request diversity. + +4. **Why did the release cause a crash loop rather than degrade safely?** + The memory limit was sized for the old application baseline and did not include sufficient headroom for the new allocation pattern. Restarted instances were also immediately exposed to queued and retried traffic, causing the cache to refill quickly. + +5. **Why was the defect not detected before or shortly after deployment?** + CI did not include memory-regression soak or load testing, and production monitoring did not alert on sustained memory utilization, abnormal restart count, or `OOMKilled` events. + +**Root cause:** A memory leak introduced in `v2.18.0`—an unbounded in-process cache—caused container memory to grow until it exceeded a limit sized for the previous baseline. The impact was compounded by retry traffic, insufficient memory headroom, absent memory and restart alerting, and the lack of memory-regression testing in CI. + +## 4. Recommendations + +| Action | Owner (role) | Priority | +|---|---|---| +| Replace the unbounded cache with a size- and TTL-bounded implementation; add eviction metrics and tests. | Application engineering lead | P0 | +| Recalculate container memory requests/limits from measured peak usage and retain operational headroom. | Platform/SRE engineer | P0 | +| Alert when per-container memory utilization exceeds 80% for five minutes. | Observability/SRE engineer | P0 | +| Alert when a workload records more than three restarts in ten minutes, and surface `OOMKilled`/exit-code-137 events. | Observability/SRE engineer | P0 | +| Implement the proposed horizontal auto-scaling policy below after load validation. | Platform/SRE engineer | P1 | +| Add sustained soak/load tests and memory-regression thresholds to CI before production promotion. | Quality engineering and application teams | P1 | +| Write and exercise an OOM/crash-loop response runbook covering diagnosis, rollback, temporary headroom, and retry control. | Incident management/SRE lead | P1 | +| Add bounded exponential backoff and jitter to applicable client and internal retries. | Application engineering team | P2 | + +## 5. Proposed Auto-scaling Policy for Ghaymah + +This is a proposed policy design for workloads deployed on Ghaymah; it does not assume a named or currently documented Ghaymah autoscaling feature. + +```yaml +policy_name: production-api-resource-scaling +mode: horizontal +replicas: + minimum: 2 + maximum: 10 # N; validate against load tests, quota, and downstream capacity +scale_out: + evaluation_window: 2m + conditions: + operator: OR + rules: + - metric: average_container_memory_utilization + threshold: "> 70%" + - metric: average_container_cpu_utilization + threshold: "> 65%" + increment: + strategy: proportional + minimum_replicas: 1 + maximum_step_percent: 50 +scale_in: + memory_threshold: "< 45%" + cpu_threshold: "< 40%" + evaluation_window: 10m + condition_operator: AND + decrement: 1 + cooldown: 10m +stabilization: + scale_out_cooldown: 2m + scale_in_cooldown: 10m +``` + +The maximum `N` is initially **10 replicas** and must be validated against measured container throughput, account capacity, and the limits of databases and other downstream services. Scale-out occurs when either average memory exceeds 70% or average CPU exceeds 65% for two continuous minutes. Scale-in requires both metrics to remain low and uses a 10-minute cooldown—within the requested 5–10 minute range—to prevent flapping. At least two replicas remain active for availability. + +Autoscaling alone does **not** fix a memory leak: every new replica runs the same defective code and will eventually leak. Horizontal scaling buys response time, preserves capacity during a gradual rise, and absorbs the retry storm while engineers mitigate the defect. It should be complemented by a properly measured memory limit with vertical headroom, bounded retries, readiness checks, and a restart policy that replaces failed containers without creating an uncontrolled hot loop. The lasting corrective action is to remove the leak. + +## 6. Early Detection with a Monitoring Approach on Ghaymah + +Because Ghaymah's public documentation does not establish specific monitoring, alerting, or autoscaling product features, the following approach uses platform-agnostic container signals. These signals can be collected from container runtime statistics and events, combined with an external monitor that calls the application's `/health` endpoint. + +### Signals and patterns + +- Plot per-container memory working set and memory utilization against the configured limit. A steady creep after a deployment, or a repeating sawtooth pattern that rises to the limit and drops when a container restarts, is an early indicator of a leak or crash loop. +- Track container restart count and restart rate by workload, release version, and container instance. +- Capture termination reason, especially `OOMKilled`, and exit code 137. +- Monitor `/health` availability and latency externally so an alert remains effective even when the application or container-level telemetry is unavailable. +- Correlate memory, restarts, HTTP error rate, latency, and deployments on the same operational dashboard. + +### Alert rules + +| Signal | Proposed rule | Purpose | +|---|---|---| +| Memory utilization | Per-container memory > 80% of limit for 5 minutes | Warn before the runtime enforces the limit. | +| Restart count | More than 3 restarts for the same workload in 10 minutes | Detect a crash loop early. | +| OOM event | Any `OOMKilled` event or exit code 137 in production | Page immediately on confirmed memory exhaustion. | +| Health availability | Two or more consecutive `/health` failures from an external monitor | Detect loss of service independently of container telemetry. | +| Memory growth | Sustained positive memory slope after a release, with no return toward baseline | Identify slow leaks before the hard threshold is reached. | + +Operational dashboards should show the current release version alongside memory utilization, limit, restart count, termination reason, health status, latency, and error rate. Alerts should route to the on-call notification channel with the workload, container, release, current memory percentage, restart count, and a link to the OOM runbook. + + + diff --git a/q3-cicd/CICD.md b/q3-cicd/CICD.md new file mode 100644 index 0000000..8a98fd1 --- /dev/null +++ b/q3-cicd/CICD.md @@ -0,0 +1,138 @@ +# CI/CD Pipeline for Ghaymah + +## Pipeline overview + +The workflow builds the API from `q1-deploy-monitor/app`, publishes two Docker Hub tags, deploys the immutable commit tag to a separate staging application, pauses for manual approval, and then deploys the exact same image to production. Promoting the same immutable `${{ github.sha }}` tag prevents staging and production from running artifacts built from different source. + +```mermaid +flowchart LR + A["Commit to main"] --> B["Build image"] + B --> C["Push SHA and latest tags
to Docker Hub"] + C --> D["Deploy staging
(automatic)"] + D --> E{"Manual approval
GitHub Environment"} + E -->|Approved| F["Deploy production"] + E -->|Rejected| G["Stop deployment"] +``` + +The workflow runs on: + +- A push to `main`. +- A manual `workflow_dispatch` run. + +It uses these GitHub Actions secrets: + +| Secret | Purpose | +|---|---| +| `DOCKERHUB_USERNAME` | Docker Hub account/namespace used to authenticate and construct the image name. | +| `DOCKERHUB_TOKEN` | Docker Hub access token used to push images. | +| `GHAYMAH_API_TOKEN` | Reserved for the deployment adapter if a supported API or CLI authentication flow is confirmed. | + +Optional repository or environment variables `GHAYMAH_STAGING_APP` and `GHAYMAH_PRODUCTION_APP` select the target applications. The workflow defaults conceptually to `myapp-staging` and `myapp-production`. + +Images are published as: + +```text +docker.io//ghaymah-api: +docker.io//ghaymah-api:latest +``` + +The SHA tag is used for deployment because it is immutable and auditable. `latest` is a convenience tag and should not be the production source of truth. + +Ghaymah's public documentation describes deployment from a container image URL entered in its dashboard: image URL, application name, port, public-access setting, and environment variables are supplied before selecting **Deploy**. A Ghaymah-hosted registry is not assumed. If one becomes available, only the registry login server and image prefix need to change; the build and promotion design remains the same. + +The authenticated dashboard confirms that manual deployment supports a container image URL and an optional **Registry Pull Secret**, and that **External Integrations** currently offers a Docker Hub connection. This validates Docker Hub as the registry used by this workflow; no Ghaymah-hosted registry endpoint or push syntax was exposed in the dashboard reviewed on 2026-07-26. + +## Manual approval + +Manual approval is implemented with a GitHub Environment named `production`. The workflow declares: + +```yaml +environment: production +``` + +Required reviewers are configured in the GitHub repository UI, not in workflow YAML: + +1. Open **Settings → Environments**. +2. Create or select the `production` environment. +3. Enable the deployment protection rule for required reviewers. +4. Add the people or teams authorized to approve production deployments. +5. Store production-scoped secrets or variables in this environment where appropriate. + +After staging succeeds, the `deploy-production` job enters a waiting state. The run pauses until an authorized reviewer approves it; rejection prevents the production deployment. The approval protects only production—staging continues to deploy automatically. + +## Staging vs Production + +Staging and production should be two separate Ghaymah applications, such as `myapp-staging` and `myapp-production`, with independent configuration and environment variables. + +| Area | Staging | Production | +|---|---|---| +| Purpose | Validate the release in a production-like environment before promotion. | Serve the live customer workload. | +| Data | Synthetic, anonymized, or disposable test data. | Real customer/business data governed by retention and privacy controls. | +| Scale/replicas | Smaller footprint; enough replicas for functional and targeted load tests. | Sized for peak traffic, resilience, and operational headroom. | +| Secrets | Staging-only credentials with limited permissions. | Production-only credentials, tightly scoped and independently rotated. | +| Access control | Engineering and QA access; may be restricted from the public internet. | Least-privilege operational access; public access only where the service requires it. | +| Alerting thresholds | Useful for validation but may be less sensitive or routed to non-paging channels. | SLO-based thresholds with paging for user-impacting failures. | +| Deploy cadence | Automatic after each successful build from `main`. | Only after staging succeeds and a reviewer approves the deployment. | +| Who can approve | No approval required for this pipeline. | Reviewers assigned to the GitHub `production` Environment. | + +Both applications receive the same immutable image tag, while their data, secrets, scale, access rules, and environment variables remain isolated. + +## Ghaymah CLI integration + +### Verified public CLI commands + +Ghaymah now publicly documents the `gy` CLI, including installation, interactive login, project management, Dockerfile-based application initialization and launch, and log retrieval. The currently documented flow is: + +```bash +curl -sSL https://cli.ghaymah.systems/install.sh | bash +source ~/.bashrc +gy version +gy auth login +gy auth status +gy resource project get +gy resource app init --project-id --name +gy resource app launch [PATH] +gy resource app logs +gy resource app update --set '' +``` + +`app init` creates `.ghaymah.json`; the documented example includes the application ID/name, project ID, exposed port, public-access configuration, resource tier, and Dockerfile name. `app launch` builds the local Dockerfile and deploys that application. + +Official sources: + +- [Ghaymah CLI documentation](https://ghaymah.systems/docs) +- [Ghaymah CLI overview](https://ghaymah.systems/cli) + +### CI integration gap + +Direct inspection of CLI version `0.0.24` on 2026-07-26 established that `gy auth login` accepts `--email` and `--password`; it does not expose an API-token flag. It also established that `gy resource app update ` accepts JSON input or dot notation through `--set`. + +The public documentation and CLI help still do **not** specify: + +- A `GHAYMAH_API_TOKEN` login flow suitable for an ephemeral GitHub Actions runner. +- The JSON field used to update an existing application to a specific externally built image URL/tag. +- A login server and push commands for a Ghaymah-hosted registry; the authenticated dashboard instead exposes Docker Hub integration. + + + +Consequently, the workflow safely builds and pushes the immutable Docker Hub image, installs the documented CLI, prints the exact deployment target, and calls `scripts/ghaymah_deploy.sh`. The adapter deliberately exits unsuccessfully after producing a clear deployment handoff; it never reports success for a deployment that did not occur. + +Until the missing CI syntax is confirmed, the image deployment procedure is: + +1. Open the target Ghaymah application in the dashboard. +2. Update its container image URL to `docker.io//ghaymah-api:`. +3. Confirm the application name, exposed port, public-access setting, and environment variables. +4. Select **Deploy** and validate application health. +5. Repeat for production only after the GitHub Environment approval. + +The staging and production workflow steps print the exact SHA-tagged image and target application, then call: + + + +```bash +bash scripts/ghaymah_deploy.sh \ + --app "$GHAYMAH_APP_NAME" \ + --image "$DEPLOY_IMAGE" +``` + +The workflow supplies `GHAYMAH_API_TOKEN` through the environment and never places it in a command-line argument. Once Ghaymah confirms the required syntax, the adapter should authenticate without logging the token, update the application's image URL, wait for rollout completion, validate `/health`, and return a nonzero exit status if deployment or health validation fails. diff --git a/q4-scalability/SCALABILITY.md b/q4-scalability/SCALABILITY.md new file mode 100644 index 0000000..30db2ec --- /dev/null +++ b/q4-scalability/SCALABILITY.md @@ -0,0 +1,114 @@ +# Scalability and Load Distribution on Ghaymah + +## 1. Architecture for 15,000 requests per second + +```mermaid +flowchart LR + U["Clients
15,000 req/s"] -->|"Resolve application hostname"| DNS["DNS"] + DNS -->|"Route requests"| CDN["CDN / edge cache
absorbs cacheable bursts"] + CDN -->|"Cache misses and dynamic requests"| LB["L7 load balancer
TLS termination and routing"] + + subgraph API["Auto-scaled stateless API fleet"] + direction TB + C1["API containers
minimum capacity: 39"] + C2["Additional containers
during scale-out / deploys"] + end + + LB -->|"Distribute dynamic traffic"| C1 + LB -->|"Sharp-load scale-out"| C2 + C1 -->|"Hot-key and session-independent reads"| REDIS["Redis cache
absorbs repeated-read bursts"] + C2 -->|"Hot-key and session-independent reads"| REDIS + C1 -->|"Writes and cache misses"| PRIMARY["PostgreSQL primary
Ghaymah Block Storage volume"] + C2 -->|"Writes and cache misses"| PRIMARY + REDIS -->|"Cache miss"| PRIMARY + PRIMARY -->|"Streaming replication"| R1["PostgreSQL read replica 1
Ghaymah Block Storage volume"] + PRIMARY -->|"Streaming replication"| R2["PostgreSQL read replica 2
Ghaymah Block Storage volume"] + C1 -->|"Read-only queries"| R1 + C2 -->|"Read-only queries"| R2 + + DNS -.-> MON["Monitoring and alerting
metrics, logs, traces, health checks"] + CDN -.-> MON + LB -.-> MON + C1 -.-> MON + C2 -.-> MON + REDIS -.-> MON + PRIMARY -.-> MON + R1 -.-> MON + R2 -.-> MON +``` + +The normal request path is **DNS → CDN/edge cache → L7 load balancer → stateless API container**. Cacheable content is answered at the CDN edge, so it does not consume API capacity. Dynamic requests reach the container fleet, where Redis absorbs repeated reads and hot-key bursts. Database writes go to the PostgreSQL primary, while eligible read-only traffic is distributed across read replicas. The monitoring component observes health, throughput, saturation, latency, errors, logs, and traces across every tier. + +The CDN and Redis cache are the first two burst buffers. They reduce the amount of work reaching the API and database, but capacity planning still assumes the API may need to handle the full 15,000 req/s. This conservative assumption prevents cache misses, invalidation events, or an unusually write-heavy workload from immediately exhausting the fleet. + +## 2. Container count calculation + +Given: + +- Peak application load: **15,000 req/s** +- Tested capacity per container: **500 req/s** +- Required headroom: **30%** + +Base container count: + +```text +15,000 req/s ÷ 500 req/s per container = 30 containers +``` + +Add 30% headroom: + +```text +30 × 1.30 = 39 containers +``` + +**Answer: 39 application containers.** + +Capacity calculations must always round fractional results up to the next whole container. The fleet should also permit at least **N+1**, or **40 active container slots**, during rolling deployments so one extra container can become ready before an old container is removed. The 30% operating headroom is reserved to absorb traffic spikes and the modeled loss of a zone without exceeding the tested 500 req/s-per-container limit. The zone distribution and failure model must be load-tested to confirm that 30% is sufficient for the actual placement topology. + +The 500 req/s figure must represent a sustainable result at acceptable latency and error rate—not a short-lived maximum from a synthetic test. Database, Redis, network, and load-balancer capacity must be tested at the same target because adding API containers cannot remove a downstream bottleneck. + +## 3. Cold-start strategy + +- **Maintain a warm replica floor.** Keep at least the calculated normal-capacity fleet available for this high-throughput tier and never scale it to zero. Retain additional warm capacity when traffic is volatile or startup time is longer than the acceptable scaling delay. +- **Use slim, pre-pulled images.** Build from a small production base image, exclude build tools and development dependencies, and keep the layer count low. Pre-pull the approved image onto worker capacity where the runtime permits it so a scale-out does not wait for a large registry download. +- **Make application boot fast.** Initialize only request-critical dependencies during startup. Lazy-load optional modules, background reports, large reference datasets, and other non-critical work after the process can safely serve traffic. +- **Enforce a readiness probe.** The load balancer must route traffic only after the process is listening and critical dependencies are usable. Readiness should remain false while caches, connection pools, or mandatory configuration are warming; liveness should be separate so slow startup is not mistaken for a dead process. +- **Scale predictively before known peaks.** Use historical demand and scheduled events to add capacity several minutes before daily peaks, campaigns, batch workloads, or announced launches. Reactive scaling then handles deviations from the forecast. +- **Use step-based scale-out.** Under a sharp rise, add several containers in one scaling action rather than one at a time. Choose step sizes from measured startup duration and traffic growth—for example, add 25% of the current fleet when utilization crosses the high threshold, subject to the maximum capacity and downstream limits. +- **Warm new instances progressively.** Ramp traffic to newly ready containers rather than sending a full share immediately. This allows connection pools and caches to warm without creating a synchronized load spike against Redis or PostgreSQL. +- **Control retries during startup.** Apply bounded exponential backoff with jitter and retry budgets so failed requests do not multiply the load while new capacity is becoming ready. + +## 4. Ghaymah Block Storage for stateful workloads + +Containers are ephemeral: their writable container filesystem may disappear when an instance is restarted, replaced, rescheduled, or scaled in. Durable state must therefore live outside the application container. Applied to Ghaymah, a block-storage volume should be attached to a stateful service and formatted or managed by that service just as a persistent disk would be. + +Appropriate uses include: + +- PostgreSQL database files and write-ahead logs. +- Durable queue or broker data when the selected queue requires a filesystem-backed store. +- User uploads when a filesystem interface is required, although object storage is generally preferable for horizontally shared upload data. + +The stateless API tier remains completely diskless. It stores no durable sessions, uploads, or business records on its local container filesystem, so any API instance can be created, replaced, or removed without data migration. + +### Attachment and scaling model + +A conventional filesystem on a block volume should be treated as a **single-writer resource** unless the storage service and filesystem explicitly support safe multi-writer attachment. Multiple API or database containers must not concurrently mount and write to the same ordinary filesystem volume. The stateful database tier scales through database-aware replication: + +1. The PostgreSQL primary owns its data volume and accepts writes. +2. Each read replica owns a separate volume containing its replicated database copy. +3. PostgreSQL replication transfers changes from the primary to the replicas. +4. Read traffic can be distributed to replicas; writes continue to use the primary. + +This design does not clone and concurrently mount one writable volume across replicas. Replication preserves database consistency at the application layer and allows each database instance to manage its own disk. + +### Backup, performance, and capacity + +Crash-consistent or application-consistent volume snapshots can provide a backup building block, but database-aware backups and restore tests are still required. A snapshot should be coordinated with PostgreSQL or combined with its write-ahead log archive so the recovery point is valid. + + + +The authenticated Ghaymah dashboard displays a supported volume-size range of **50 MiB minimum to 10 GiB maximum** and allows a volume to be attached from an application's advanced deployment options. Volume sizing must account for the live dataset, indexes, temporary files, write-ahead logs, maintenance operations, expected growth, and free-space safety margin. Performance planning must account for sustained and burst IOPS, throughput, latency, queue depth, and the read/write mix. Load tests should verify database latency at expected peak traffic rather than selecting capacity from size alone. + + + +Block Storage makes container replacement compatible with durable state, but it does not replace database replication, tested backups, point-in-time recovery, or a documented failover procedure. diff --git a/q5-mithal-dashboard/Dockerfile b/q5-mithal-dashboard/Dockerfile new file mode 100644 index 0000000..5205571 --- /dev/null +++ b/q5-mithal-dashboard/Dockerfile @@ -0,0 +1,41 @@ +FROM python:3.12-slim + +# curl is used by the HEALTHCHECK below. +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PORT=8080 \ + WEB_ROOT=/app/dashboard \ + METRICS_FILE=/app/dashboard/data/metrics.json + +WORKDIR /app + +# Dependencies first so editing the collector or dashboard doesn't reinstall them. +COPY collector/requirements.txt ./collector/requirements.txt +RUN pip install --no-cache-dir -r collector/requirements.txt + +# Application: the collector and the static dashboard it feeds. +COPY collector/collect.py ./collector/collect.py +COPY dashboard/index.html ./dashboard/index.html +COPY start.sh ./start.sh + +# Tolerate CRLF line endings if the repo was checked out on Windows. +RUN sed -i 's/\r$//' ./start.sh && chmod +x ./start.sh + +# Non-root user; it must be able to write metrics into the served directory. +RUN useradd --create-home --uid 10001 appuser \ + && mkdir -p /app/dashboard/data \ + && chown -R appuser:appuser /app +USER appuser + +EXPOSE 8080 + +# The dashboard page is the liveness signal for the whole container. +HEALTHCHECK --interval=60s --timeout=5s --start-period=15s --retries=3 \ + CMD curl -f http://localhost:8080/index.html || exit 1 + +CMD ["./start.sh"] diff --git a/q5-mithal-dashboard/README.md b/q5-mithal-dashboard/README.md new file mode 100644 index 0000000..e2dd6c4 --- /dev/null +++ b/q5-mithal-dashboard/README.md @@ -0,0 +1,211 @@ +# Q5 — Monitoring dashboard for mithal.space + +A single container that continuously measures the availability and performance of +**https://mithal.space** and serves a live dashboard of the results on port 8080. + +``` +q5-mithal-dashboard/ +├── collector/ +│ ├── collect.py # measures latency, uptime, DNS, TLS, search — every 60s +│ └── requirements.txt # requests (pinned); everything else is stdlib +├── dashboard/ +│ ├── index.html # self-contained dashboard (Chart.js from CDN) +│ └── data/metrics.json # rolling 48h JSON array, written by the collector +├── start.sh # entrypoint: collector in background + static server +├── Dockerfile +└── README.md +``` + +**Why `data/` lives inside `dashboard/`:** the dashboard directory *is* the web +root, so the page fetches `data/metrics.json` from its own origin. One container, +one port, no CORS, no API layer. + +--- + +## 1. What is collected + +Every 60 seconds `collector/collect.py` appends one record to the JSON array: + +```json +{ + "ts": "2026-07-26T15:46:40.381676+00:00", + "up": true, + "code": 200, + "latency_ms": 848.58, + "dns_ms": 0.67, + "ssl_days_left": 50, + "search_ms": 1056.08 +} +``` + +| Field | How it is measured | +|---|---| +| `latency_ms` | timed `GET https://mithal.space`, 10 s timeout, redirects followed | +| `up` | `true` when the status code is **200–399** | +| `code` | the HTTP status code — `null` when the connection itself failed | +| `dns_ms` | timed `socket.getaddrinfo("mithal.space", 443)` | +| `ssl_days_left` | TLS handshake to `mithal.space:443`, cert `notAfter` parsed → days remaining | +| `search_ms` | timed `GET https://mithal.space/search?q=test` (`null` if it errors or returns ≥ 400) | + +Every measurement is independent: a failure records `null` for that field only and +never aborts the run or crashes the loop. Records older than **48 hours** are +pruned on each write, and the file is written atomically (temp file + rename) so +the dashboard never reads a half-written array. + +**Configuration** — constants at the top of `collect.py`, all overridable by env var: + +| Variable | Default | Meaning | +|---|---|---| +| `TARGET_URL` | `https://mithal.space` | site under test | +| `SEARCH_URL` | `https://mithal.space/search?q=test` | search endpoint to time | +| `INTERVAL_S` | `60` | seconds between collections | +| `TIMEOUT_S` | `10` | per-request timeout | +| `RETENTION_HOURS` | `48` | how much history to keep | +| `METRICS_FILE` | `dashboard/data/metrics.json` | output path | + +--- + +## 2. Run locally (without Docker) + +```bash +cd q5-mithal-dashboard +python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r collector/requirements.txt +``` + +One-shot collection (useful as a smoke test or under cron): + +```bash +python collector/collect.py --once +``` + +Continuous collection every 60 s — leave this running: + +```bash +python collector/collect.py +``` + +In a second terminal, serve the dashboard (the `data/` directory must be inside +the served root, which it is): + +```bash +cd q5-mithal-dashboard/dashboard && python -m http.server 8080 +``` + +Open . + +> Opening `index.html` straight from disk does **not** work — browsers block +> `fetch()` over `file://`. The page detects this and shows an explanatory banner +> instead of failing silently. + +--- + +## 3. The dashboard + +| Element | Detail | +|---|---| +| Status badge | green `UP` / red `DOWN` from the newest record, with HTTP code and timestamp | +| Uptime tile | `up_checks / total_checks × 100` over the **last 24 h**, one decimal | +| TLS card | "*X* days remaining" — green > 30, yellow 8–30, red ≤ 7 (`Expired` at ≤ 0) | +| Latest response | newest `latency_ms`, with `search_ms` and `dns_ms` underneath | +| Chart | `latency_ms` (solid blue) and `search_ms` (dashed purple) for the **last hour**; failed checks draw gaps, red points mark down checks | +| Table | last 10 checks — time, ✅/❌, code, latency, DNS, search | +| Refresh | re-fetches every 60 s; last-updated clock in the header | + +Empty, missing, or corrupt data renders a "no data yet" state on every tile plus a +banner explaining what to do — it never throws. + +To point the page elsewhere, edit the one constant at the top of the ` + + + +
+ +
+

mithal.space · monitoring

+
Last updated —
+
+ + + +
+
+
Current status
+
NO DATA
+
Waiting for the first collection…
+
+ +
+
Uptime — last 24h
+
—
+
— up / — checks
+
+ +
+
TLS certificate
+
—
+
days remaining
+ +
+ +
+
Latest response
+
—
+
search — · dns —
+
+
+ +
+
+
Response time — last hour
+
+
+
+ page latency + search latency +
+
+ + +
+
+ +
+
Last 10 checks
+
+ + + + + + + + +
TimeStatusCodeLatencyDNSSearch
+
+ +
+ +
+ Data source: + Auto-refresh every 60s · 48h retention +
+
+ + + + diff --git a/q5-mithal-dashboard/start.sh b/q5-mithal-dashboard/start.sh new file mode 100644 index 0000000..3cb5e47 --- /dev/null +++ b/q5-mithal-dashboard/start.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Entrypoint: run the collector in the background, serve the dashboard in the +# foreground. The collector writes into the served directory, so the page can +# fetch data/metrics.json from the same origin — no CORS, no second service. +set -eu + +WEB_ROOT="${WEB_ROOT:-/app/dashboard}" +PORT="${PORT:-8080}" + +mkdir -p "$WEB_ROOT/data" + +# Keep the collector alive: if it ever exits, restart it after a short pause +# rather than leaving the dashboard serving frozen data. +( + while true; do + python /app/collector/collect.py || echo "[start.sh] collector exited, restarting in 10s" >&2 + sleep 10 + done +) & +COLLECTOR_PID=$! + +# Stop both processes on SIGTERM/SIGINT so the container shuts down promptly. +trap 'kill "$COLLECTOR_PID" 2>/dev/null || true; exit 0' TERM INT + +echo "[start.sh] serving $WEB_ROOT on port $PORT" +exec python -m http.server "$PORT" --directory "$WEB_ROOT" --bind 0.0.0.0 diff --git a/screenshots/ghaymah-project-created.png b/screenshots/ghaymah-project-created.png new file mode 100644 index 0000000..214caf5 Binary files /dev/null and b/screenshots/ghaymah-project-created.png differ diff --git a/scripts/ghaymah_deploy.sh b/scripts/ghaymah_deploy.sh new file mode 100644 index 0000000..f30f315 --- /dev/null +++ b/scripts/ghaymah_deploy.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Deployment adapter for the GitHub Actions workflow. +# +# Ghaymah publicly documents `gy resource app launch` for deploying a +# local Dockerfile/.ghaymah.json project. It does not currently document a +# non-interactive API-token login command. The current binary exposes +# email/password login flags and a generic `app update` command, but its public +# help does not identify an external-image field. This adapter therefore +# validates and prints the exact deployment handoff, then fails deliberately +# instead of reporting a deployment that did not happen. + +app="" +image="" + +usage() { + echo "Usage: $0 --app --image " >&2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --app) + [[ $# -ge 2 ]] || { usage; exit 64; } + app="$2" + shift 2 + ;; + --image) + [[ $# -ge 2 ]] || { usage; exit 64; } + image="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + usage + exit 64 + ;; + esac +done + +[[ -n "$app" && -n "$image" ]] || { usage; exit 64; } + +echo "Ghaymah deployment handoff" +echo " Application: $app" +echo " Image URL: $image" +echo +echo "Update the target application's container image URL to the immutable tag" +echo "shown above in the Ghaymah dashboard, then validate its health endpoint." +echo +echo "Automated deployment is intentionally blocked because the public CLI docs" +echo "do not specify API-token authentication or the external-image field for" +echo "'gy resource app update'. Replace this adapter only with confirmed syntax." + +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + { + echo "### Ghaymah deployment requires confirmed integration" + echo + echo "- Application: \`$app\`" + echo "- Image: \`$image\`" + echo + echo "The image was built and pushed, but no deployment was claimed. Confirm" + echo "the supported API/CLI command or update this image URL in the dashboard." + } >> "$GITHUB_STEP_SUMMARY" +fi + +exit 78