Complete Ghaymah cloud assessment

2026-07-26 19:45:13 +03:00
commit 8bc563fa6b
27 changed files with 4152 additions and 0 deletions
@@ -0,0 +1,101 @@
 name: Build and deploy
 on:
  push:
    branches:
      - main
  workflow_dispatch:
 permissions:
  contents: read
 env:
  IMAGE_NAME: docker.io/${{ secrets.DOCKERHUB_USERNAME }}/ghaymah-api
 jobs:
  build-push:
    name: Build and push image
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
      - name: Log in to Docker Hub
        uses: docker/login-action@v3
        with:
          registry: docker.io
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Build and push immutable and latest tags
        uses: docker/build-push-action@v6
        with:
          context: q1-deploy-monitor/app
          file: q1-deploy-monitor/app/Dockerfile
          push: true
          tags: |
            ${{ env.IMAGE_NAME }}:${{ github.sha }}
            ${{ env.IMAGE_NAME }}:latest
  deploy-staging:
    name: Deploy to staging
    runs-on: ubuntu-latest
    needs: build-push
    env:
      DEPLOY_IMAGE: docker.io/${{ secrets.DOCKERHUB_USERNAME }}/ghaymah-api:${{ github.sha }}
      GHAYMAH_APP_NAME: ${{ vars.GHAYMAH_STAGING_APP }}
      GHAYMAH_API_TOKEN: ${{ secrets.GHAYMAH_API_TOKEN }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
      - name: Show staging deployment target
        run: |
          echo "Image to deploy: ${DEPLOY_IMAGE}"
          echo "Ghaymah app: ${GHAYMAH_APP_NAME:-myapp-staging}"
      - name: Install the documented Ghaymah CLI
        run: |
          # VERIFY: Official installation command as published at https://ghaymah.systems/docs on 2026-07-26.
          curl -sSL https://cli.ghaymah.systems/install.sh | bash
          echo "$HOME/ghaymah/bin" >> "$GITHUB_PATH"
      - name: Deploy staging image
        shell: bash
        run: |
          # VERIFY: The CLI is documented, but non-interactive token auth and external-image update syntax are not. Replace this adapter when confirmed.
          bash scripts/ghaymah_deploy.sh \
            --app "${GHAYMAH_APP_NAME:-myapp-staging}" \
            --image "${DEPLOY_IMAGE}"
  deploy-production:
    name: Deploy to production
    runs-on: ubuntu-latest
    needs: deploy-staging
    environment: production
    env:
      DEPLOY_IMAGE: docker.io/${{ secrets.DOCKERHUB_USERNAME }}/ghaymah-api:${{ github.sha }}
      GHAYMAH_APP_NAME: ${{ vars.GHAYMAH_PRODUCTION_APP }}
      GHAYMAH_API_TOKEN: ${{ secrets.GHAYMAH_API_TOKEN }}
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
      - name: Show production deployment target
        run: |
          echo "Image to deploy: ${DEPLOY_IMAGE}"
          echo "Ghaymah app: ${GHAYMAH_APP_NAME:-myapp-production}"
      - name: Install the documented Ghaymah CLI
        run: |
          # VERIFY: Official installation command as published at https://ghaymah.systems/docs on 2026-07-26.
          curl -sSL https://cli.ghaymah.systems/install.sh | bash
          echo "$HOME/ghaymah/bin" >> "$GITHUB_PATH"
      - name: Deploy production image
        shell: bash
        run: |
          # VERIFY: The CLI is documented, but non-interactive token auth and external-image update syntax are not. Replace this adapter when confirmed.
          bash scripts/ghaymah_deploy.sh \
            --app "${GHAYMAH_APP_NAME:-myapp-production}" \
            --image "${DEPLOY_IMAGE}"
@@ -0,0 +1,28 @@
 # Python
 __pycache__/
 *.py[cod]
 .pytest_cache/
 .mypy_cache/
 .ruff_cache/
 .venv/
 venv/
 # Local environment and secrets
 .env
 .env.*
 !.env.example
 *.pem
 *.key
 # Editors and operating systems
 .vscode/
 .idea/
 .DS_Store
 Thumbs.db
 # Temporary and log files
 *.tmp
 *.log
 # Local Ghaymah CLI configuration may contain account or deployment metadata.
 .ghaymah.json
@@ -0,0 +1,133 @@
 # Ghaymah Test — Step-by-Step Checklist
 Work through this top to bottom. Each step is tagged with who does it:
 **[YOU]** = manual work · **[OPUS]** = prompt Opus 5 · **[SOL]** = prompt Sol/Codex
 Steps inside the same phase that have different tags can run at the same time.
 ---
 ## PHASE 0 — Setup & Research (Day 1 morning, ~2–3h)
 - [ ] **0.1 [YOU]** Create a Git repo `ghaymah-test` with this folder structure:
  ```
  q1-deploy-monitor/{app,monitor,dashboard}
  q2-postmortem/
  q3-cicd/
  q4-scalability/
  q5-mithal-dashboard/{collector,data,dashboard}
  README.md
  ```
 - [ ] **0.2 [YOU]** Create/log in to your **ghaymah.systems** account.
 - [ ] **0.3 [YOU]** Explore and take notes + screenshots on:
  - [ ] Container platform (how to create a service, set port, get public URL)
  - [ ] Container Registry (registry URL, how to log in/push)
  - [ ] ghaymah CLI (install command, auth command, deploy command)
  - [ ] Monitoring tools (metrics, graphs, alert rules)
  - [ ] Block Storage (how volumes attach, limits)
  > These notes go into every agent prompt — Q2/Q3/Q4 must cite real Ghaymah features.
 - [ ] **0.4 [YOU]** Open **mithal.space** in a browser, run a search, and copy the search URL pattern (e.g. `/search?q=test`). Save it for step 4.1.
 - [ ] **0.5 [YOU]** Decide stack: **Python + FastAPI** (assumed below).
 ---
 ## PHASE 1 — Dispatch both agents (Day 1 afternoon)
 Send these two prompts at the same time — the tracks don't share files.
 - [ ] **1.1 [OPUS]** Prompt Opus with: full Q1 text + your platform notes + repo layout. Ask for:
  - [ ] `q1-deploy-monitor/app/main.py` — FastAPI with `GET /`, `GET /health` (returns `{"status":"ok","uptime_s":...,"timestamp":...}`), `GET /metrics` (in-memory request counter via middleware)
  - [ ] `app/requirements.txt` (fastapi, uvicorn)
  - [ ] `app/Dockerfile` — `python:3.12-slim`, non-root user, layer-cached COPY order, `EXPOSE 8080`, `HEALTHCHECK` hitting `/health`, plus `.dockerignore`
  - [ ] `monitor/monitor.py` — loop every 30s: GET `$APP_URL/health` (5s timeout), append `{"ts","status","code","latency_ms","requests"}` to `monitor/data/checks.json`; "down" on exception; alert print after 3 consecutive fails
  - [ ] `dashboard/index.html` — single file, no build step: green/red status badge, latency line chart (Chart.js CDN), request-count tile, fetches `checks.json`, auto-refresh 30s
 - [ ] **1.2 [SOL]** Prompt Sol with: full Q2 text + your monitoring-tool notes. Ask for `q2-postmortem/POSTMORTEM.md` containing:
  - [ ] Summary (45-min outage, OOMKilled exit 137, impact, severity)
  - [ ] Minute-by-minute timeline table (memory creep → OOMKill → crash-loop → alert → limit raise + rollback → recovery)
  - [ ] Root cause via 5-Whys (memory leak in release X + undersized limit + no memory alerting/CI check)
  - [ ] Recommendations table (owner + priority)
  - [ ] Auto-scaling policy: scale out at mem>70% or CPU>65% for 2 min, min 2 replicas, scale-in cooldown 5–10 min — **and** an explicit note that HPA alone doesn't fix a leak
  - [ ] Early-detection section citing real Ghaymah monitoring features; every platform claim marked `<!-- VERIFY -->`
 ---
 ## PHASE 2 — Q1 goes live (Day 2 morning)
 - [ ] **2.1 [YOU]** Review Opus's Q1 output. Build & test locally:
  ```
  docker build -t ghaymah-api ./q1-deploy-monitor/app
  docker run -p 8080:8080 ghaymah-api
  ```
  - [ ] `http://localhost:8080/health` returns 200 JSON
  - [ ] `/metrics` counter increments
 - [ ] **2.2 [YOU]** Push image to ghaymah Container Registry (`docker login` → `docker tag` → `docker push`).
 - [ ] **2.3 [YOU]** Create the container service on ghaymah.systems (port 8080), note the **public URL**, screenshot the running service + `/health` response.
 - [ ] **2.4 [YOU]** Start `monitor.py` with `APP_URL=<public URL>` and **leave it running** so real data accumulates before submission.
 - [ ] **2.5 [YOU]** Open the Q1 dashboard against the growing `checks.json` — verify badge, chart, counter all render.
 - [ ] **2.6 [OPUS]** Send any deployment fixes back to Opus, then immediately give it Q5 (step 4.1) — don't wait.
 ---
 ## PHASE 3 — CI/CD (Day 2 morning, parallel with Phase 2)
 - [ ] **3.1 [SOL]** Prompt Sol with: full Q3 text + your CLI/registry notes. Ask for:
  - [ ] `.github/workflows/deploy.yml` — trigger on push to `main` + `workflow_dispatch`; jobs:
    - `build-push`: checkout → `docker/login-action` (secrets `GHAYMAH_REGISTRY_USER`/`TOKEN`) → `docker/build-push-action` tagging `:${{ github.sha }}` and `:latest`
    - `deploy-staging` (needs build-push): deploy via ghaymah CLI
    - `deploy-production` (needs staging): bound to GitHub Environment `production` (this is the manual-approval gate)
  - [ ] `q3-cicd/CICD.md` — staging vs production section (data, scale, secrets, access, alerting, cadence + promotion flow diagram) and ghaymah CLI integration section (install, `auth login` with `GHAYMAH_API_TOKEN` secret, deploy command); placeholders marked `<!-- VERIFY -->`
 - [ ] **3.2 [YOU]** Push repo to GitHub. In repo settings:
  - [ ] Add secrets: `GHAYMAH_REGISTRY_USER`, `GHAYMAH_REGISTRY_TOKEN`, `GHAYMAH_API_TOKEN`
  - [ ] Settings → Environments → create `production` → add yourself as **required reviewer**
 - [ ] **3.3 [YOU]** Trigger the workflow, let it pause at the production gate, **screenshot the approval prompt**, then approve.
 ---
 ## PHASE 4 — Q5 build & deploy (Day 2 afternoon)
 - [ ] **4.1 [OPUS]** Prompt Opus with: full Q5 text + the mithal.space search URL from 0.4. Ask for:
  - [ ] `collector/collect.py` — every 60s (plus `--once` flag) measure:
    - Latency: timed `GET https://mithal.space`
    - Uptime: up if status 200–399
    - SSL: `ssl`/`socket` cert `notAfter` → days remaining
    - DNS: timed `socket.getaddrinfo`
    - Search: timed request to the search URL (configurable)
    - Append `{"ts","up","code","latency_ms","dns_ms","ssl_days_left","search_ms"}` to `data/metrics.json`, prune >48h
  - [ ] `dashboard/index.html` — uptime % tile (24h), latency line chart (last hour, Chart.js), SSL card color-coded (green>30d / yellow 8–30 / red≤7), last-10-checks table, auto-refresh 60s; same visual style as Q1
  - [ ] `Dockerfile` — one container serving the dashboard statically **and** running the collector in the background (shell entrypoint launching both), collector writes into the served `data/` dir
 - [ ] **4.2 [YOU]** Test locally (`--once` first, then the loop; open the dashboard). Fix issues via Opus.
 - [ ] **4.3 [YOU]** Push image + deploy to ghaymah exactly like 2.2–2.3. Screenshot the live dashboard URL.
 - [ ] **4.4 [YOU]** Leave it running so the 24h uptime % and hourly chart fill with real data.
 ---
 ## PHASE 5 — Q4 scalability doc (Day 2 afternoon, parallel with Phase 4)
 - [ ] **5.1 [SOL]** Prompt Sol with: full Q4 text + your Block Storage notes. Ask for `q4-scalability/SCALABILITY.md`:
  - [ ] Mermaid architecture diagram: DNS → CDN → L7 load balancer → stateless API fleet (auto-scaled) → Redis cache → DB primary+replicas on Block Storage; monitoring on the side; annotate the 15k req/s path
  - [ ] Container math shown explicitly: `15,000 ÷ 500 = 30 → ×1.30 = 39 containers`, plus notes on rounding up and N+1 for rolling deploys
  - [ ] Cold-start strategy: warm pool / min replicas, slim pre-pulled images, readiness-probe gating, predictive scaling before peaks, step scale-out policies
  - [ ] Block Storage section: ephemeral containers vs persistent volumes, one-writer semantics, snapshots/backups, IOPS; API tier stays diskless — `<!-- VERIFY -->` markers on platform specifics
 ---
 ## PHASE 6 — Verification & polish (Day 3 morning)
 - [ ] **6.1 [YOU]** Search the whole repo for `<!-- VERIFY -->` and resolve every one against real Ghaymah docs (fix CLI commands, registry URL, monitoring feature names, Block Storage details).
 - [ ] **6.2 [YOU]** Insert real values everywhere: live app URL (Q1), registry image name (Q3), live dashboard URL (Q5).
 - [ ] **6.3 [YOU]** Add all screenshots to the repo: Q1 service + `/health`, Q3 approval gate, Q5 live dashboard.
 - [ ] **6.4 [SOL]** Send Sol any corrections needed in Q2/Q3/Q4 docs from your verification pass.
 - [ ] **6.5 [OPUS]** Ask Opus for final dashboard polish now that real data exists (empty-state handling, chart scaling).
 ---
 ## PHASE 7 — Final review & submit (Day 3 afternoon)
 - [ ] **7.1 [YOU]** Rubric pass — check every numbered sub-requirement one by one:
  - [ ] Q1: Dockerfile · deployed on ghaymah · `/health` · 30s monitor script · dashboard (status, latency, request count)
  - [ ] Q2: postmortem (summary, timeline, root cause, recommendations) · auto-scaling policy · early-detection with Ghaymah tools
  - [ ] Q3: build+push workflow · manual approval before production · staging vs production explained · CLI integration documented
  - [ ] Q4: architecture diagram · container count (39) with math · cold-start strategy · Block Storage explanation
  - [ ] Q5: collector (latency, uptime, SSL, DNS, search) every minute · CSV/JSON storage · dashboard (uptime %, latency chart, SSL, last 10 checks) · deployed on ghaymah
 - [ ] **7.2 [YOU]** Write `README.md` as an index: one section per question, links to every file, live URLs, screenshots.
 - [ ] **7.3 [YOU]** Confirm both dashboards show real accumulated data (not empty charts).
 - [ ] **7.4 [YOU]** Final commit + push. Submit.
@@ -0,0 +1,228 @@
 # Finalization Runbook
 Everything that can be done without your credentials is done. This file is the
 copy-paste path through what remains. Work top to bottom.
 > Note: written while another agent (Codex) was editing files in this repo. The
 > line numbers in §5 are a snapshot — re-run the grep before trusting them.
 ---
 ## 0. What is already verified (no need to redo)
 Verified locally on 2026-07-26 with Docker 29.5.3 (linux/amd64):
 | Item | Result |
 |---|---|
 | `docker build` Q1 (`q1-deploy-monitor/app`) | builds clean → `ghaymah-api:latest`, 249 MB |
 | `docker build` Q5 (`q5-mithal-dashboard`) | builds clean → `mithal-monitor:latest`, 200 MB |
 | Q1 container | `/health` → 200 `{"status":"ok","uptime_s":11.254,...}`; `/metrics` counter increments; Docker healthcheck reports **healthy** |
 | Q5 container | `start.sh` launches collector + static server; collector wrote a real record inside the container (`up:true, code:200, ssl_days_left:50`); dashboard rendered at `/index.html` with badge, uptime tile, TLS card, chart and table; healthcheck **healthy** |
 | Q1 monitor | up/down transitions, `ALERT` after exactly 3 consecutive failures, `--once` exit codes |
 | Q5 collector | live measurements against mithal.space; unreachable-host run records all-nulls without crashing |
 **Both images are already built on this machine.** You can go straight to `docker push`.
 ---
 ## 1. Set your Docker Hub username once
 Everything below reuses this. Run in PowerShell:
 ```powershell
 $DH = "yourdockerhubusername"   # <-- edit this line only
 ```
 ---
 ## 2. Push both images (10 min)
 ```powershell
 docker login
 ```
 ```powershell
 docker tag ghaymah-api:latest "docker.io/$DH/ghaymah-api:latest"
 docker tag mithal-monitor:latest "docker.io/$DH/mithal-monitor:latest"
 docker push "docker.io/$DH/ghaymah-api:latest"
 docker push "docker.io/$DH/mithal-monitor:latest"
 ```
 Then, on hub.docker.com, confirm **both repositories are Public** — Ghaymah pulls
 them anonymously. A private repo is the single most common cause of a deployment
 that never leaves "pulling".
 ---
 ## 3. Deploy both apps on Ghaymah (30 min)
 Same form twice:
 | Field | Q1 | Q5 |
 |---|---|---|
 | Container Image URL | `docker.io/<DH>/ghaymah-api:latest` | `docker.io/<DH>/mithal-monitor:latest` |
 | Application Name | `ghaymah-api` | `mithal-monitor` |
 | Port Number | `8080` | `8080` |
 | Public Access | enabled | enabled |
 | Env vars (optional) | `APP_NAME=ghaymah-api`, `APP_VERSION=1.0.0` | `INTERVAL_S=60` |
 Record both public URLs — you need them in §5.
 Verify immediately:
 ```powershell
 curl.exe -f "<Q1_URL>/health"
 ```
 Then open `<Q5_URL>/index.html` in a browser (it will show 1–2 checks at first).
 ### Start the Q1 monitor the moment Q1 is live
 This is the long pole — the Q1 dashboard is only convincing with hours of history.
 Leave this window open for the rest of the assessment:
 ```powershell
 $env:APP_URL="<Q1_URL>"; python q1-deploy-monitor\monitor\monitor.py
 ```
 Q5 needs nothing — its collector runs inside the deployed container.
 ---
 ## 4. Resolve the Q3 blocker (the CLI question) — 20 min
 **What I established from public sources** (so you don't repeat the search):
 - Install: `curl -sSL https://cli.ghaymah.systems/install.sh | bash`
 - The installer downloads `gy-{platform}-{arch}` from `https://cli.ghaymah.systems`
  (override with `SERVER_URL`) and installs to `$HOME/ghaymah/bin/gy`.
 - The only env var the installer honours is `SERVER_URL`. **No token env var exists
  in the installer.**
 - Documented commands: `gy auth login` (browser-based), `gy auth status`,
  `gy resource project get`, `gy resource project create --set .name=my-new-project`,
  `gy resource application init --project-id <ID>`, `gy resource application launch`,
  `gy resource application logs`.
 - `.ghaymah.json` fields: `id`, `name`, `projectId`, `ports[].expose/.number`,
  `publicAccess.enabled/.domain`, `resourceTier`, `dockerFileName`, `env`, `domains`.
 - **Nothing public documents non-interactive auth, a registry hostname, or updating
  an application to an external image tag.** `/docs/cli` returns 404; the changelog
  and product pages list "container registry" and "CI/CD" as features with no syntax.
 **Run these three commands after `gy auth login`** — they are the fastest way to
 settle it, and their output is what I need to finish the workflow:
 ```bash
 gy auth login --help
 gy --help
 gy resource application --help
 ```
 Look specifically for: a `--token` / `--api-key` flag, any `GHAYMAH_*` env var, a
 `gy resource application update`/`set-image` subcommand, or a `gy registry` group.
 Also check the authenticated dashboard for an **API tokens / service accounts**
 page and for the registry hostname.
 Paste whatever those print back to me and I will finish `deploy.yml`,
 `scripts/ghaymah_deploy.sh` and `CICD.md` in one pass.
 If it turns out no non-interactive path exists, that is a legitimate finding —
 the honest write-up (deploy job documented, adapter exits 78 rather than faking
 success, manual promotion step described) scores better than invented syntax.
 Do **not** invent a command to make the pipeline look green.
 ---
 ## 5. Replace the placeholders
 Snapshot of what is unfilled (re-run the grep after Codex finishes):
 ```powershell
 Select-String -Path (Get-ChildItem -Recurse -Include *.md,*.yml,*.sh -Path . | Where-Object FullName -notmatch '\\\.git\\') -Pattern 'MY_DOCKERHUB_USER|the public URL Ghaymah assigns|<!-- VERIFY|<PROJECT_ID>'
 ```
 | Placeholder | Where | Replace with |
 |---|---|---|
 | `<MY_DOCKERHUB_USER>` | `q1-deploy-monitor/README.md` (5×), `q5-mithal-dashboard/README.md` (5×) | your Docker Hub user |
 | `<the public URL Ghaymah assigns>` | `q1-deploy-monitor/README.md` (6×), `q5-mithal-dashboard/README.md` (1×) | the two live URLs |
 | `<PROJECT_ID>` | `q3-cicd/CICD.md:91`, root `README.md:22` | your Ghaymah project id |
 | `<!-- VERIFY -->` × 5 | `q2-postmortem/POSTMORTEM.md:133`, `q3-cicd/CICD.md:111,125`, `q4-scalability/SCALABILITY.md:108,112` | see §6 |
 Bulk replacement (run **after** Codex is done, from the repo root):
 ```powershell
 $DH="yourdockerhubusername"; $Q1="https://your-q1-url"; $Q5="https://your-q5-url"
 Get-ChildItem -Recurse -Include *.md -File | Where-Object FullName -notmatch '\\\.git\\' | ForEach-Object {
  (Get-Content $_.FullName -Raw) `
    -replace '<MY_DOCKERHUB_USER>', $DH `
    -replace '<the public URL Ghaymah assigns>', $Q1 |
  Set-Content $_.FullName -Encoding utf8
 }
 ```
 That sets the Q1 URL everywhere; then fix the single Q5 occurrence in
 `q5-mithal-dashboard/README.md` by hand (it is the `/index.html` line).
 ---
 ## 6. The five remaining VERIFY areas
 Each needs one look at the **authenticated** dashboard. What to check, and what to
 do with the answer:
 | # | File | Question to answer in the dashboard |
 |---|---|---|
 | 1 | `q2-postmortem/POSTMORTEM.md:133` | Does Ghaymah show per-container memory/CPU metrics, restart counts, and configurable alert rules + notification channels? Screenshot the monitoring page. |
 | 2 | `q3-cicd/CICD.md:111` | Non-interactive auth for CI — settled by §4. |
 | 3 | `q3-cicd/CICD.md:125` | External-image update syntax — settled by §4. |
 | 4 | `q4-scalability/SCALABILITY.md:108` | Block Storage: snapshots supported? retention/encryption/restore? |
 | 5 | `q4-scalability/SCALABILITY.md:112` | Block Storage: size limits, resize, IOPS/throughput, attachment/access modes, zone binding. |
 Rule for all five: if the platform confirms it, state it plainly and drop the
 marker. If the platform does **not** expose it, rewrite the sentence to describe
 the general requirement and say the platform-specific limits were not documented
 at the time of writing — then drop the marker. **No `VERIFY` string may survive in
 the submitted repo.**
 ---
 ## 7. Screenshots to capture
 Save under `docs/screenshots/` with these names so the READMEs can link them:
 - `q1-service-running.png` — the Ghaymah dashboard showing the running Q1 app
 - `q1-health-response.png` — browser or terminal showing `/health` → 200 JSON
 - `q1-monitor-dashboard.png` — the Q1 dashboard with a populated latency chart
 - `q3-approval-gate.png` — the GitHub Actions run paused on the production reviewer prompt
 - `q5-live-dashboard.png` — the deployed Q5 dashboard with ≥ 1 h of history
 - `ghaymah-monitoring.png` — optional, supports the Q2 detection section
 ---
 ## 8. GitHub, secrets, pipeline
 1. Create the repo, add the remote, push.
 2. Settings → Secrets and variables → Actions: add the secrets named in
   `.github/workflows/deploy.yml` (Docker Hub user/token, plus whatever §4 settles
   for Ghaymah).
 3. Settings → Environments → `production` → **Required reviewers: yourself**.
 4. Run the workflow → let it pause → screenshot the gate → approve.
 Decide before pushing whether `PLAN.md`, `PROMPTS.md` and `CHECKLIST.md` belong in
 the submission. They document your internal agent workflow; graders don't need
 them. Either delete them or list them in `.gitignore`.
 ---
 ## 9. Final gate before submitting
 ```powershell
 Select-String -Path (Get-ChildItem -Recurse -Include *.md,*.yml,*.sh -File | Where-Object FullName -notmatch '\\\.git\\') -Pattern 'VERIFY|MY_DOCKERHUB_USER|the public URL Ghaymah assigns'
 ```
 Must return nothing. Then:
 - [ ] Both apps reachable at their public URLs
 - [ ] Q1 dashboard chart has hours of points; Q5 uptime tile computed over real 24 h data
 - [ ] All screenshots committed and linked from the root README
 - [ ] Every numbered sub-requirement of Q1–Q5 ticked (CHECKLIST.md §7.1)
 - [ ] Final commit pushed
@@ -0,0 +1,218 @@
 # Ghaymah Internship Test — Implementation Plan
 **Scope:** 5 questions · 72 hours · 100 points (20 each)
 **Execution model:** Two agents working in parallel — **Sol** (Codex) and **Opus 5** — with you as the integrator who does the manual platform steps (accounts, deployments, secrets).
 ---
 ## 0. Ground Rules & Repo Setup (do this first, yourself — 30 min)
 **Purpose:** Both agents need a shared structure so their outputs merge cleanly without conflicts. Doing this before dispatching work prevents rework.
 **How:**
 1. Create one Git repo (e.g. `ghaymah-test`) with this layout:
 ```
 ghaymah-test/
 ├── q1-deploy-monitor/
 │   ├── app/                 # API + Dockerfile
 │   ├── monitor/             # monitoring script
 │   └── dashboard/           # HTML/CSS/JS dashboard
 ├── q2-postmortem/
 │   └── POSTMORTEM.md
 ├── q3-cicd/
 │   ├── .github/workflows/   # (copied to repo root at the end)
 │   └── CICD.md
 ├── q4-scalability/
 │   └── SCALABILITY.md       # diagram + math + strategy
 ├── q5-mithal-dashboard/
 │   ├── collector/           # metrics script
 │   ├── data/                # CSV/JSON output
 │   └── dashboard/
 └── README.md                # index of all answers
 ```
 2. Sign up / log in to **ghaymah.systems**, explore: Container platform, Container Registry, CLI docs, monitoring tools, Block Storage docs. **Take notes/screenshots** — Q2, Q3, Q4 answers must reference *real* Ghaymah features, not generic cloud talk. This research is the single highest-value manual task: graders will notice platform-specific accuracy.
 3. Decide the stack once: **Python + FastAPI** for both APIs (Q1 and Q5 collector share language, smaller surface area) — or Node/Express if you prefer. The plan below assumes Python.
 **Why manual:** account creation, credentials, and platform exploration cannot be delegated to agents (and shouldn't be — never give agents your passwords/API keys; store tokens as env vars/GitHub secrets yourself).
 ---
 ## Parallelization Map (who does what)
 | Track | Agent | Work | Rationale |
 |---|---|---|---|
 | A | **Opus 5** | Q1 (app + Dockerfile + monitor + dashboard), Q5 (collector + dashboard) | Heaviest coding volume, frontend polish, end-to-end coherence — Opus is strongest at multi-file builds and design-quality dashboards |
 | B | **Sol** | Q3 (CI/CD workflow + docs), Q2 (postmortem), Q4 (scaling doc + diagram) | Mostly structured writing + one YAML workflow — well-bounded, spec-driven tasks that run independently of Track A |
 | — | **You** | Ghaymah account, deployments, registry creds, GitHub secrets, real screenshots, final review | Anything requiring credentials or the live platform |
 The two tracks share **zero files**, so they can run truly simultaneously. Merge point: Day 2 evening, when you deploy Track A's artifacts and paste real values (URLs, registry names) into Track B's docs.
 **Prompting tip:** give each agent the full question text *plus* your Ghaymah platform notes from step 0, and tell them the repo layout above so paths match.
 ---
 ## Q1 — Deploy & Monitor an App on Ghaymah (20 pts)
 ### 1.1 The API app + `/health` endpoint — **Opus**
 **Purpose:** The deliverable everything else in Q1 hangs off. `/health` is the standard liveness contract that orchestrators and your monitor script both consume.
 **How:** Small FastAPI app with:
 - `GET /` — hello/info route
 - `GET /health` — returns `{"status":"ok","uptime_s":...,"timestamp":...}` with HTTP 200. Keep it dependency-free (no DB check) so it reflects process liveness only.
 - `GET /metrics` — returns an in-memory request counter + simple stats (this feeds the dashboard's "عدد الطلبات" requirement). Implement with a middleware that increments a counter per request.
 **Implementation:** `app/main.py`, `app/requirements.txt` (fastapi, uvicorn). Run with `uvicorn main:app --host 0.0.0.0 --port 8080`.
 ### 1.2 Dockerfile — **Opus**
 **Purpose:** Shows you understand image hygiene, not just "it runs".
 **How:** Multi-stage or slim single-stage:
 - `python:3.12-slim` base, non-root user, `COPY requirements.txt` before code (layer caching), `EXPOSE 8080`, `HEALTHCHECK CMD curl -f http://localhost:8080/health || exit 1`, `CMD uvicorn...`.
 - Add `.dockerignore`.
 **Implementation check:** you build locally: `docker build -t ghaymah-api . && docker run -p 8080:8080 ghaymah-api`, hit `/health`.
 ### 1.3 Deploy to ghaymah.systems — **You (manual)**
 **Purpose:** The 20 points require a *live* deployment; agents can't do this.
 **How:** Per platform docs — typically: push image to ghaymah Container Registry (`docker login <registry>`, `docker tag`, `docker push`), then create a container service in the dashboard/CLI pointing at the image, set port 8080, note the public URL. **Screenshot the running service and the `/health` response** for your submission.
 ### 1.4 Monitoring script (every 30s) — **Opus**
 **Purpose:** Demonstrates operational thinking — a poller that produces machine-readable history the dashboard can consume.
 **How:** `monitor/monitor.py`:
 - Loop: every 30s, `GET <APP_URL>/health` with timeout (e.g. 5s), record `timestamp, status ("up"/"down"), http_code, response_ms`; also pull `/metrics` for request count.
 - Append each check to `monitor/data/checks.json` (or JSONL) — this file **is the dashboard's data source**, so agree the schema up front:
  ```json
  {"ts":"2026-07-26T12:00:00Z","status":"up","code":200,"latency_ms":42,"requests":1337}
  ```
 - Config via env var `APP_URL`. Handle exceptions → status "down", latency null. Optional: alert line to stdout when 3 consecutive failures.
 ### 1.5 Dashboard (HTML/CSS/JS) — **Opus**
 **Purpose:** Shows the three required metrics: status, response time, request count.
 **How:** Single static page `dashboard/index.html` (inline CSS/JS, zero build step):
 - Status badge (green/red) from latest check
 - Line chart of `latency_ms` (Chart.js from CDN, or hand-rolled SVG if you want zero dependencies)
 - Request counter tile
 - Fetches `checks.json` (served next to it, or the monitor writes into the dashboard folder) and refreshes every 30s with `setInterval`.
 **Deliverable checklist Q1:** Dockerfile ✔ live URL ✔ `/health` ✔ monitor script ✔ dashboard ✔ screenshots ✔
 ---
 ## Q2 — OOMKilled Postmortem (20 pts) — **Sol**
 **Purpose:** Pure documentation question testing incident-response maturity. No code; graded on structure, realism, and platform-specific recommendations.
 **How:** `q2-postmortem/POSTMORTEM.md` in the classic blameless format:
 1. **Summary** — 45-min outage, repeated OOMKilled restarts (exit code 137), impact (error rate / downtime %), severity level.
 2. **Timeline** — invent a realistic minute-by-minute table: memory creep begins → first OOMKill → crash-loop (each restart re-accumulates memory faster under retry traffic) → alert fires → engineer raises memory limit + rolls back the leaking release → recovery. Timestamps, actor, action.
 3. **Root cause** — pick something concrete and defensible, e.g. a memory leak introduced in release X (unbounded in-process cache) combined with a memory limit sized for the old baseline; 5-Whys chain down to "no memory regression check in CI, no memory alerting."
 4. **Recommendations** — table with owner + priority: right-size limits, fix leak, add memory alerts at 80%, add auto-scaling (links to part 2), load-test with soak tests, add runbook.
 5. **Auto-scaling policy design** — a concrete policy for Ghaymah containers:
   - Horizontal: scale out at avg memory > 70% or CPU > 65% for 2 min; min 2 / max N replicas; scale-in cooldown 5–10 min to avoid flapping.
   - Explicitly explain *why HPA alone doesn't fix a leak* (leaks eat any replica count — scaling buys time for the fix, plus restart policy / vertical headroom). This nuance is what separates a top answer.
 6. **Early detection with Ghaymah monitoring** — reference the actual tools you found in step 0: memory-usage graphs per container, alert rules (memory > 80% for 5 min, restart-count > 3/10 min, OOMKilled event alerts), dashboards + notification channels.
 **Implementation:** one well-formatted Markdown file; optionally a small Mermaid timeline/graph. **You** later swap in accurate Ghaymah tool names from your notes.
 ---
 ## Q3 — CI/CD Pipeline (20 pts) — **Sol**
 **Purpose:** Tests GitHub Actions fluency + registry integration + release discipline (manual gate).
 ### 3.1 Workflow — build & push to ghaymah Container Registry
 **How:** `.github/workflows/deploy.yml`:
 - Trigger: `push` to `main` (+ `workflow_dispatch`).
 - Job `build-push`: checkout → `docker/login-action` against the ghaymah registry using `secrets.GHAYMAH_REGISTRY_USER/TOKEN` → `docker/build-push-action` tagging `registry.ghaymah.systems/<user>/app:${{ github.sha }}` and `:latest`.
 - Job `deploy-staging` (needs build): deploy via ghaymah CLI to the staging service.
 - Job `deploy-production` (needs staging): bound to a GitHub **Environment** named `production` with **required reviewers** — this is the manual-approval mechanism (job pauses until a human approves in the Actions UI). Sol should document that the environment + reviewers are configured in repo Settings → Environments, since that part isn't in YAML.
 ### 3.2 Staging vs Production explanation
 **How:** Section in `CICD.md`: purpose (validation vs. live users), differences (data, scale/replicas, secrets, access control, alerting thresholds, deploy cadence), promotion flow diagram `commit → build → staging (auto) → approval → production`.
 ### 3.3 ghaymah CLI integration docs
 **How:** `CICD.md` section: install CLI, `ghaymah auth login` with an API token stored as `GHAYMAH_API_TOKEN` GitHub secret, deploy/update-image command, how the workflow calls it in a step. **You** verify exact CLI command names from real docs and correct Sol's draft — flag every placeholder Sol writes with `<!-- VERIFY -->` so nothing invented ships.
 **You (manual):** create the GitHub repo, add the secrets, create the `production` environment with yourself as reviewer, run the pipeline once and screenshot the approval gate.
 ---
 ## Q4 — Scalability & Load Balancing (20 pts) — **Sol**
 **Purpose:** Architecture reasoning + arithmetic + platform storage knowledge. Fully deterministic — ideal Sol task.
 ### 4.1 Architecture diagram (15,000 req/s)
 **How:** Mermaid diagram in `SCALABILITY.md` (renders on GitHub): DNS → CDN/edge cache → Load Balancer (L7) → stateless API container fleet (auto-scaled) → cache layer (Redis) → DB (primary + replicas) → Block Storage for stateful pieces; plus monitoring/queue components. Annotate where the 15k req/s flows and what absorbs bursts.
 ### 4.2 Container count math — show the work
 ```
 15,000 req/s ÷ 500 req/s per container = 30 containers
 +30% headroom: 30 × 1.30 = 39 containers
 ```
 State the answer plainly (**39**), then add operational notes: round up, N+1 for rolling deploys, and that headroom covers spikes + AZ loss.
 ### 4.3 Cold-start strategy
 **How:** bullet strategy: keep a warm pool / min-replicas floor, pre-pull & slim images (small base, fewer layers), lazy-load nothing critical at boot, readiness probe gating so LB never routes to a cold container, predictive/scheduled scaling ahead of known peaks, gradual (step) scale-out policies.
 ### 4.4 ghaymah Block Storage for stateful workloads
 **How:** explain: containers are ephemeral → attach Block Storage volumes for databases/queues/uploads; persistence across restarts/reschedules; one-writer-per-volume semantics (so stateful services scale differently than stateless API tier); snapshots/backups; IOPS considerations. Tie back to the diagram (DB nodes mount Block Storage; API tier stays diskless). **You** correct against real Ghaymah Block Storage docs.
 ---
 ## Q5 — mithal.space Monitoring Dashboard (20 pts) — **Opus**
 **Purpose:** The most integrated build: multi-metric collector + persisted history + richer dashboard + deployment. Assign to Opus because it shares patterns (and dashboard code style) with Q1 — one agent keeps them consistent.
 ### 5.1 Collector script (every minute) — `collector/collect.py`
 **How (one Python script, stdlib + `requests`):**
 - **Latency:** `requests.get("https://mithal.space", timeout=10)` — measure elapsed ms.
 - **Uptime:** same request's status code → up if `200 ≤ code < 400`.
 - **SSL:** `ssl` + `socket` — open TLS connection to port 443, read cert `notAfter`, compute days remaining.
 - **DNS:** time `socket.getaddrinfo("mithal.space", 443)` in ms.
 - **Search response:** since mithal.space is a search engine — send a query (inspect the site first to find the search URL pattern, e.g. `/search?q=test` or its API endpoint) and time the response. **You** should check the actual URL format in a browser and give it to Opus; have Opus make it configurable.
 - Scheduling: `while True: run(); sleep(60)` **plus** support one-shot mode (`--once`) so it can also run under cron / a scheduler.
 ### 5.2 Storage
 **How:** append one JSON object per check to `data/metrics.json` (array) or JSONL; keep a rolling window (e.g. last 24–48h, prune older) so the file stays small. Schema fixed up-front (same discipline as Q1):
 ```json
 {"ts": "...", "up": true, "code": 200, "latency_ms": 120,
 "dns_ms": 18, "ssl_days_left": 143, "search_ms": 210}
 ```
 ### 5.3 Dashboard — `dashboard/index.html`
 **How:** static page, fetches `metrics.json`, renders:
 - **Uptime % (24h):** `checks_up / checks_total * 100` over last 24h — big number tile.
 - **Latency line chart (last hour):** Chart.js line of `latency_ms` (optionally overlay `search_ms`).
 - **SSL card:** days remaining, color-coded (green > 30, yellow 8–30, red ≤ 7).
 - **Last-10-checks table:** time, status ✅/❌, code, latency, DNS, search.
 - Auto-refresh every 60s. Same visual language as Q1's dashboard (Opus keeps them consistent).
 ### 5.4 Deploy dashboard to ghaymah — **You + Opus**
 **How:** Opus writes a tiny Dockerfile that serves the dashboard **and** runs the collector in the same container (simplest: FastAPI/nginx serving static files + collector as background process writing into the served `data/` dir — a small `supervisord` or a shell entrypoint launching both). You push and deploy exactly as in Q1, screenshot the live URL.
 ---
 ## Timeline (72h budget, comfortably front-loaded)
 | When | You | Opus (Track A) | Sol (Track B) |
 |---|---|---|---|
 | **Day 1 AM** | Repo setup, Ghaymah account + docs research, notes | — | — |
 | **Day 1 PM** | Feed notes + prompts to both agents | Q1 app, Dockerfile, monitor, dashboard | Q2 postmortem full draft |
 | **Day 2 AM** | Build & deploy Q1, registry setup, GitHub secrets | Q5 collector + dashboard | Q3 workflow + CICD.md |
 | **Day 2 PM** | Deploy Q5, run pipeline, capture approval screenshot | Fix anything found in deployment | Q4 scalability doc |
 | **Day 3 AM** | Verify every `<!-- VERIFY -->` against real Ghaymah docs, insert real URLs/screenshots | Polish dashboards with real data | Revisions from your review |
 | **Day 3 PM** | Final README index, cross-check rubric (all sub-items × 5 questions), submit | — | — |
 ## Quality Gates (your final pass)
 1. Every question's *numbered sub-requirements* are individually answered — graders score per item.
 2. No invented Ghaymah feature names survive — everything platform-specific is verified against real docs.
 3. Both apps are **live** with URLs + screenshots in the README.
 4. Monitor/collector scripts actually ran long enough to produce real data in the dashboards (start them Day 2 so charts aren't empty at submission).
 5. Repo README links every deliverable — one-click grading experience.
@@ -0,0 +1,223 @@
 # Copy-Paste Agent Prompts + Ghaymah Info to Collect
 > **PART C (added after research): the actual notes to paste — see bottom of this file. Docs turned out sparse, so Part C replaces Part A.**
 ## PART A — What to grab from ghaymah.systems FIRST (~20 min)
 Log in and write down these exact values. Paste them into the `[GHAYMAH NOTES]` block of each prompt below.
 1. **Registry URL** — the container registry hostname (e.g. `registry.ghaymah.systems` or similar) and how login works (username + token? access key?).
 2. **How to deploy a container** — dashboard steps or CLI command to create a service from an image, set the port, and get the public URL. Note what the public URLs look like.
 3. **CLI** — exact install command, exact auth/login command, exact deploy/update command (copy them verbatim from the docs page).
 4. **Monitoring** — what metrics/graphs exist per container (CPU? memory? restarts?), whether alert rules exist and what they can trigger on, notification channels.
 5. **Auto-scaling** — does the platform have built-in autoscaling settings? What knobs (min/max replicas, CPU/memory thresholds)?
 6. **Block Storage** — product name as they call it, how a volume attaches to a container, size limits, snapshot support.
 7. **Environments** — can you run two services (staging + production)? Any built-in env concept, or just two separate services?
 8. Also: open **mithal.space**, do a search, copy the result URL pattern (e.g. `https://mithal.space/search?q=test`).
 Screenshot everything as you go — you need screenshots for the submission anyway.
 ---
 ## PART B — The 5 prompts
 Fill `[GHAYMAH NOTES: ...]` with the relevant items from Part A before sending. Send Prompt 1 (Opus) and Prompt 2 (Sol) simultaneously; then 3 & 4; then 5.
 ---
 ### PROMPT 1 → OPUS (Q1: app + Dockerfile + monitor + dashboard)
 ```
 Build a complete deliverable for this task, in the folder q1-deploy-monitor/:
 TASK: Deploy and monitor a simple API app on the Ghaymah cloud container platform.
 Create exactly these files:
 1. app/main.py — Python FastAPI app with:
   - GET /  → basic info JSON
   - GET /health → {"status":"ok","uptime_s":<float>,"timestamp":<iso8601>} with HTTP 200, no external dependencies
   - GET /metrics → {"requests_total":<int>, "started_at":<iso8601>} using an in-memory counter incremented by a middleware on every request
   - Runs with uvicorn on 0.0.0.0:8080
 2. app/requirements.txt — fastapi + uvicorn, pinned versions.
 3. app/Dockerfile — python:3.12-slim, non-root user, COPY requirements.txt and pip install BEFORE copying code (layer caching), EXPOSE 8080, HEALTHCHECK curl -f http://localhost:8080/health, CMD uvicorn. Also app/.dockerignore.
 4. monitor/monitor.py — Python monitoring script:
   - Reads APP_URL from env var
   - Every 30 seconds: GET $APP_URL/health with 5s timeout, and GET /metrics
   - Appends one JSON object per check to monitor/data/checks.json (a JSON array):
     {"ts":"<iso8601>","status":"up"|"down","code":<int|null>,"latency_ms":<float|null>,"requests":<int|null>}
   - Any exception/timeout → status "down"
   - Prints an ALERT line after 3 consecutive failures
   - Also supports --once flag for a single check
 5. dashboard/index.html — ONE self-contained file (inline CSS/JS, Chart.js from CDN allowed) that fetches ../monitor/data/checks.json (path configurable in a const at top) and shows:
   - Big status badge, green UP / red DOWN, from the latest check
   - Line chart of latency_ms over time
   - Request-count tile from the latest check
   - Last-updated timestamp, auto-refresh every 30s
   - Handle empty/missing data gracefully (show "no data yet", don't crash)
   Make it look clean and professional (dark theme is fine).
 Also write q1-deploy-monitor/README.md: how to build/run the Docker image locally, how to push it to the Ghaymah registry and deploy it, how to run the monitor, how to open the dashboard.
 [GHAYMAH NOTES: <paste registry URL, deploy steps, public URL format here>]
 ```
 ---
 ### PROMPT 2 → SOL (Q2: postmortem)
 ```
 Write q2-postmortem/POSTMORTEM.md — a professional, blameless postmortem for this scenario:
 SCENARIO: An application on the Ghaymah cloud platform was down for 45 minutes due to repeated OOMKilled container restarts.
 Required sections:
 1. Summary — one paragraph: duration 45 min, cause OOMKilled (exit code 137) crash loop, user impact (est. error rate / full downtime), severity SEV-2.
 2. Timeline — a table with minute-by-minute realistic entries (invent plausible timestamps): gradual memory growth after a release → first OOMKill → crash loop worsened by retry traffic → alert fires → engineer investigates → mitigation (memory limit raised + rollback of the leaking release) → full recovery at minute 45. Columns: time, event, actor/action.
 3. Root cause — 5-Whys analysis. Root: a memory leak introduced in the latest release (unbounded in-process cache), combined with a memory limit sized for the old baseline, no memory-usage alerting, and no memory regression testing in CI.
 4. Recommendations — table with columns: action, owner (role), priority (P0/P1/P2). Include: fix the leak, right-size memory limits, memory alert at 80% for 5 min, restart-count alert, add auto-scaling, soak/load tests in CI, write a runbook.
 5. Auto-scaling policy for Ghaymah — a concrete policy spec: horizontal scaling, scale OUT when avg memory > 70% OR CPU > 65% sustained 2 min; min 2 replicas, max N; scale-in cooldown 5–10 min to prevent flapping. Then an explicit honest paragraph: why autoscaling alone does NOT fix a memory leak (every replica leaks; scaling buys time and absorbs the retry storm while the leak is fixed) — plus vertical headroom and restart policies as complements.
 6. Early detection with Ghaymah monitoring — how to catch this before an outage using the platform's monitoring: per-container memory graphs (watch the sawtooth/creep pattern), alert rules (memory > 80% for 5 min, restarts > 3 in 10 min, OOMKilled events), dashboards and notification channels. Wrap any claim about a specific Ghaymah feature name in <!-- VERIFY --> comments so I can check it against the real docs.
 Tone: professional SRE postmortem, markdown tables, no fluff.
 [GHAYMAH NOTES: <paste what monitoring/alerting/autoscaling features you saw here>]
 ```
 ---
 ### PROMPT 3 → SOL (Q3: CI/CD)
 ```
 Create two files for a CI/CD deliverable targeting the Ghaymah cloud platform:
 1. .github/workflows/deploy.yml — GitHub Actions workflow:
   - Triggers: push to main, and workflow_dispatch
   - Job build-push: checkout → docker/login-action to the Ghaymah container registry using secrets GHAYMAH_REGISTRY_USER and GHAYMAH_REGISTRY_TOKEN → docker/build-push-action building q1-deploy-monitor/app, tagging both <REGISTRY>/<user>/ghaymah-api:${{ github.sha }} and :latest
   - Job deploy-staging (needs: build-push): installs the ghaymah CLI, authenticates with secret GHAYMAH_API_TOKEN, deploys/updates the STAGING service to the new image tag
   - Job deploy-production (needs: deploy-staging): environment: production — this is the manual approval gate — then same deploy against the PRODUCTION service
   - Mark every Ghaymah-specific command with a # VERIFY comment
 2. q3-cicd/CICD.md with:
   - Pipeline overview + a mermaid diagram: commit → build+push → staging (auto) → manual approval → production
   - "Manual approval" section: explain it's implemented via a GitHub Environment named production with required reviewers, configured in Settings → Environments (not in YAML), and the run pauses until approved
   - "Staging vs Production" section: purpose of each, differences table (data: synthetic vs real; scale/replicas; secrets; access control; alerting thresholds; deploy cadence; who can approve)
   - "Ghaymah CLI integration" section: install, auth login with API token, the deploy/update command, and how the workflow steps call it. Wrap uncertain command names in <!-- VERIFY -->.
 [GHAYMAH NOTES: <paste exact registry URL + exact CLI install/auth/deploy commands here>]
 ```
 ---
 ### PROMPT 4 → SOL (Q4: scalability)
 ```
 Write q4-scalability/SCALABILITY.md answering this task about the Ghaymah cloud platform:
 1. Architecture diagram (mermaid, renders on GitHub) for an app receiving 15,000 req/s:
   DNS → CDN/edge cache → L7 load balancer → auto-scaled fleet of stateless API containers → Redis cache → PostgreSQL primary + read replicas (volumes on Ghaymah Block Storage) → and a monitoring/alerting component observing everything. Annotate the request flow and where bursts get absorbed (CDN + cache).
 2. Container count calculation — show the arithmetic explicitly:
   15,000 req/s ÷ 500 req/s per container = 30 containers
   +30% headroom: 30 × 1.30 = 39 containers
   Answer: 39. Add notes: always round up; keep N+1 extra for rolling deploys; headroom also covers traffic spikes and a zone failure.
 3. Cold-start strategy for new containers, as concrete bullets: minimum warm replica floor (never scale to zero for this tier), slim pre-pulled images (small base, few layers) for fast pulls, fast app boot (lazy-load non-critical work), readiness probe so the load balancer only routes to warmed containers, predictive/scheduled scale-up before known peaks, step-based scale-out (add several at once under sharp load).
 4. Ghaymah Block Storage for stateful workloads: containers are ephemeral so state must live on attached volumes; use Block Storage for the database, queues, uploads; one-writer-per-volume semantics mean the stateful tier scales via replication (primary/replica) not by cloning volumes; snapshots for backup; IOPS/size considerations; the API tier stays completely diskless so it can scale freely. Wrap platform-specific claims in <!-- VERIFY -->.
 [GHAYMAH NOTES: <paste Block Storage + autoscaling details here>]
 ```
 ---
 ### PROMPT 5 → OPUS (Q5: mithal.space monitoring)
 ```
 Build a monitoring system for the website https://mithal.space (a search engine), in q5-mithal-dashboard/:
 1. collector/collect.py — Python (stdlib + requests only):
   Every 60 seconds (loop mode; also support --once) measure:
   - latency_ms: timed GET https://mithal.space (10s timeout)
   - up: true if status code 200–399
   - code: the status code (null on connection failure)
   - ssl_days_left: open an ssl socket to mithal.space:443, read the cert notAfter, compute days remaining
   - dns_ms: timed socket.getaddrinfo("mithal.space", 443)
   - search_ms: timed GET to the search endpoint — make the URL a config constant, default: <PASTE THE SEARCH URL PATTERN YOU FOUND, e.g. https://mithal.space/search?q=test>
   Append one object per run to data/metrics.json (JSON array):
   {"ts":"<iso8601 utc>","up":bool,"code":int|null,"latency_ms":float|null,"dns_ms":float|null,"ssl_days_left":int|null,"search_ms":float|null}
   Prune entries older than 48h on each write. Never crash on a failed check — record nulls.
 2. dashboard/index.html — ONE self-contained file (Chart.js CDN allowed) fetching data/metrics.json (path const at top):
   - Uptime % tile over the last 24h: up_checks / total_checks × 100, one decimal
   - Line chart of latency_ms for the LAST HOUR (optionally search_ms as second series)
   - SSL card: "X days remaining", green if >30, yellow 8–30, red ≤7
   - Table of the last 10 checks: time, ✅/❌, code, latency, dns, search
   - Auto-refresh every 60s, graceful empty state, clean professional look consistent with a dark ops-dashboard style
 3. Dockerfile (in q5-mithal-dashboard/) — a single container that BOTH serves the dashboard as static files on port 8080 AND runs the collector in the background writing into the served directory. Simplest approach: python:3.12-slim, a start.sh entrypoint that launches collect.py in the background then runs python -m http.server 8080 (or uvicorn static serving) from the dashboard directory, with data/ inside the served path so the dashboard can fetch metrics.json.
 4. README.md: run locally, build the image, deploy to Ghaymah.
 [GHAYMAH NOTES: <paste deploy steps here>]
 ```
 ---
 ## PART C — Ready [GHAYMAH NOTES] blocks (paste these as-is)
 Research result: Ghaymah docs only document **deploy-from-image-URL** (image URL + app name + port + public access toggle + env vars → Deploy). No registry, CLI, monitoring, autoscaling, or block-storage details are documented. Strategy: use **Docker Hub** as the registry, and have agents label undocumented platform specifics as **"Proposed design"** instead of inventing features. mithal.space search URL confirmed: `https://mithal.space/search?q=<query>`.
 ### Paste into PROMPT 1 (Opus, Q1):
 ```
 [GHAYMAH NOTES]
 - Ghaymah deploys containers from a public image URL. Deployment flow (dashboard): enter Container Image URL (e.g. docker.io/<user>/ghaymah-api:latest) → set Application Name → set Port Number (must match the EXPOSEd port) → enable Public Access → add Environment Variables → click Deploy.
 - So in the README, document: build locally → docker push to Docker Hub (docker.io/<MY_DOCKERHUB_USER>/ghaymah-api) → deploy on Ghaymah by pasting that image URL with port 8080 and Public Access enabled.
 - Ghaymah's docs don't document the public URL format; write "<the public URL Ghaymah assigns>" as a placeholder.
 ```
 ### Paste into PROMPT 2 (Sol, Q2):
 ```
 [GHAYMAH NOTES]
 - Ghaymah's public docs do not document monitoring/alerting or autoscaling features. IMPORTANT: do NOT invent Ghaymah feature names. Instead:
  - Section 5 (auto-scaling policy): the task literally asks to DESIGN a policy for the platform — present it as a proposed policy design ("Proposed auto-scaling policy for Ghaymah"), which is exactly what's asked.
  - Section 6 (early detection): frame as "monitoring approach on Ghaymah" using platform-agnostic container signals (memory %, restart count, OOMKilled events, exit code 137) and note these can be collected via an external monitor hitting /health plus container runtime stats, since that's verifiable. Keep <!-- VERIFY --> markers only where you reference a platform capability.
 ```
 ### Paste into PROMPT 3 (Sol, Q3):
 ```
 [GHAYMAH NOTES]
 - Ghaymah deploys from a container image URL entered in its dashboard (image URL + app name + port + public access + env vars → Deploy). No Ghaymah-hosted registry or CLI is documented publicly.
 - Therefore: build the workflow to push to Docker Hub (docker/login-action with secrets DOCKERHUB_USERNAME / DOCKERHUB_TOKEN, image docker.io/<user>/ghaymah-api:${{ github.sha }} and :latest). Add a note: "if a Ghaymah-hosted registry is available, only the login server and image prefix change."
 - For the deploy jobs: since no CLI is documented, implement deploy-staging and deploy-production as jobs that (a) print the exact image tag to deploy and (b) call a placeholder script scripts/ghaymah_deploy.sh marked # VERIFY, and document in CICD.md that per current docs the deploy step is updating the image URL/tag in the Ghaymah dashboard for the app — with the CLI section written as "integration guide (to be confirmed against Ghaymah CLI docs)".
 - Staging vs production on Ghaymah: two separate deployed apps, e.g. myapp-staging and myapp-production, each with its own env vars; the workflow deploys staging automatically and production only after the GitHub Environment approval.
 ```
 ### Paste into PROMPT 4 (Sol, Q4):
 ```
 [GHAYMAH NOTES]
 - Ghaymah's public docs don't detail Block Storage or autoscaling specifics. Write section 4 as a correct general explanation of block storage for stateful container workloads applied to Ghaymah ("Ghaymah Block Storage"), and keep <!-- VERIFY --> only on hard specifics (size limits, snapshot support). The architecture/math/cold-start sections are platform-agnostic — no changes needed.
 ```
 ### Paste into PROMPT 5 (Opus, Q5):
 ```
 [GHAYMAH NOTES]
 - Search endpoint CONFIRMED working: https://mithal.space/search?q=test (returns a results page). Use that as the default search URL constant.
 - Deployment: Ghaymah deploys from a public image URL (dashboard: image URL + app name + port 8080 + Public Access enabled + env vars → Deploy). README should say: push image to Docker Hub, then deploy by pasting docker.io/<MY_DOCKERHUB_USER>/mithal-monitor:latest into Ghaymah with port 8080.
 ```
 ### Your remaining manual to-dos (updated)
 1. Make/confirm a **Docker Hub** account; create repos `ghaymah-api` and `mithal-monitor` (public is simplest for Ghaymah to pull).
 2. In GitHub: secrets `DOCKERHUB_USERNAME`, `DOCKERHUB_TOKEN` (access token from Docker Hub → Account Settings → Security); environment `production` with you as required reviewer.
 3. In the Ghaymah dashboard, while deploying, **look for tabs/menus the docs didn't cover** (Registry, CLI, Metrics/Logs, Scaling, Volumes). If any exist, screenshot them and resolve the matching <!-- VERIFY --> markers — instant answer upgrades for Q2/Q3/Q4.
 4. Screenshots needed: Q1 app running + /health JSON, Q3 workflow paused at approval, Q5 live dashboard.
@@ -0,0 +1,83 @@
 # Ghaymah Cloud Technical Assessment
 This repository contains the five deliverables for the Ghaymah cloud platform assessment.
 ## Deliverables
 | Question | Deliverable | Status |
 |---|---|---|
 | Q1 — Deploy and monitor an API | [Application, monitor, dashboard, and deployment guide](q1-deploy-monitor/README.md) | Implementation complete; live Ghaymah deployment evidence pending |
 | Q2 — OOMKilled postmortem | [POSTMORTEM.md](q2-postmortem/POSTMORTEM.md) | Complete |
 | Q3 — CI/CD pipeline | [Workflow](.github/workflows/deploy.yml) and [CICD.md](q3-cicd/CICD.md) | Build/push and approval gate complete; automated Ghaymah image update awaiting a documented non-interactive command |
 | Q4 — Scalability | [SCALABILITY.md](q4-scalability/SCALABILITY.md) | Complete |
 | Q5 — mithal.space monitoring | [Collector and dashboard](q5-mithal-dashboard/) | Implementation present; live deployment evidence pending |
 ## Current verified Ghaymah information
 The current public Ghaymah documentation establishes the following:
 - The official CLI is installed with `curl -sSL https://cli.ghaymah.systems/install.sh | bash`.
 - The executable is named `gy`; the current binary reports its release with `gy version`.
 - Authentication uses `gy auth login`. The current CLI exposes `--email` and `--password` flags, but does not expose an API-token flag.
 - `gy resource app init --project-id <PROJECT_ID> --name <APP_NAME>` creates `.ghaymah.json`.
 - `gy resource app launch [PATH]` builds and deploys the application described by the local Dockerfile and `.ghaymah.json`.
 - `gy resource app logs` retrieves application logs.
 - `gy resource app update <APP_ID>` accepts JSON input or dot-notation updates, but the public help does not identify a supported external-image field.
 - The documented `.ghaymah.json` example includes the application ID/name, project ID, exposed port, public access, resource tier, and Dockerfile name.
 - Ghaymah lists a container registry, CI/CD, monitoring, autoscaling, APIs, and Block Storage as products or capabilities, but the public pages reviewed for this submission do not expose enough operational syntax or limits to use their hard specifics safely.
 - The authenticated deployment form supports either a Git repository or container image URL, an optional registry pull secret, instance size, application name, port, public access, custom domain, environment variables, and attached storage volumes.
 - The authenticated volume form displays a supported size range of 50 MiB to 10 GiB.
 - The authenticated External Integrations page exposes Docker Hub connection fields; no Ghaymah-hosted registry endpoint or push instructions were visible.
 Official references:
 - [Ghaymah CLI documentation](https://ghaymah.systems/docs)
 - [Ghaymah CLI overview](https://ghaymah.systems/cli)
 - [Ghaymah products](https://ghaymah.systems/products)
 - [Ghaymah changelog](https://ghaymah.systems/changelog)
 ## Local quick checks
 ### Q1 API
 ```bash
 python -m pip install -r q1-deploy-monitor/app/requirements.txt
 python -m uvicorn main:app --app-dir q1-deploy-monitor/app --host 0.0.0.0 --port 8080
 ```
 Then open:
 - `http://localhost:8080/health`
 - `http://localhost:8080/metrics`
 - `http://localhost:8080/docs`
 ### Q1 monitor and dashboard
 ```bash
 APP_URL=http://localhost:8080 python q1-deploy-monitor/monitor/monitor.py --once
 python -m http.server 8000 --directory q1-deploy-monitor
 ```
 Open `http://localhost:8000/dashboard/index.html`.
 ### Q5 collector
 ```bash
 python -m pip install -r q5-mithal-dashboard/collector/requirements.txt
 python q5-mithal-dashboard/collector/collect.py --once
 ```
 ## Work that requires account access
 The following cannot be completed safely from source code alone:
 1. Create the Ghaymah projects/applications and obtain their IDs and public URLs.
 2. Push the Docker images using the owner's registry credentials.
 3. Deploy Q1 and Q5 and capture the required screenshots.
 4. Configure GitHub secrets and the `production` Environment with required reviewers.
 5. Confirm the supported non-interactive Ghaymah authentication mechanism for GitHub Actions.
 6. Confirm the command/API that updates an existing Ghaymah application to a specific external image tag.
 7. Run the workflow, approve production, and capture the approval evidence.
 8. Confirm monitoring/autoscaling controls and Block Storage limits in the authenticated dashboard before removing remaining `VERIFY` markers.
 No credentials should be committed to this repository. Use GitHub Secrets and Ghaymah's secret-management controls for sensitive values.
@@ -0,0 +1,262 @@
 # Q1 — Deploy & Monitor an API on Ghaymah
 A minimal FastAPI service, containerised and deployed on the **Ghaymah** container
 platform, plus a polling monitor and a static dashboard that visualises its
 availability, response time and request count.
 ```
 q1-deploy-monitor/
 ├── app/
 │   ├── main.py            # FastAPI app: /, /health, /metrics
 │   ├── requirements.txt   # pinned fastapi + uvicorn
 │   ├── Dockerfile         # python:3.12-slim, non-root, HEALTHCHECK
 │   └── .dockerignore
 ├── monitor/
 │   ├── monitor.py         # polls /health + /metrics every 30s (stdlib only)
 │   └── data/checks.json   # created on first run — the dashboard's data source
 ├── dashboard/
 │   └── index.html         # single self-contained page (Chart.js from CDN)
 └── README.md
 ```
 ---
 ## 1. The API
 | Method | Path | Response |
 |---|---|---|
 | `GET` | `/` | service name, version, endpoint list, start time |
 | `GET` | `/health` | `{"status":"ok","uptime_s":12.34,"timestamp":"2026-07-26T15:36:50.283265+00:00"}` — HTTP 200 |
 | `GET` | `/metrics` | `{"requests_total":42,"started_at":"<iso8601>"}` |
 | `GET` | `/docs` | interactive OpenAPI docs (FastAPI built-in) |
 `/health` performs **no** downstream checks (no DB, no network) — it reports
 process liveness only, so a failing check always means "restart me", which is
 exactly the signal an orchestrator's health probe should act on.
 `/metrics` is backed by an in-memory counter incremented by an HTTP middleware on
 every request. It is per-process and deliberately resets on restart — a counter
 that drops to zero in the dashboard is a visible signal that the container was
 restarted or redeployed.
 Environment variables (all optional): `APP_NAME`, `APP_VERSION`, `PORT` (default `8080`).
 ### Run locally without Docker
 ```bash
 cd q1-deploy-monitor/app
 python -m venv .venv && source .venv/bin/activate   # Windows: .venv\Scripts\activate
 pip install -r requirements.txt
 python main.py            # or: uvicorn main:app --host 0.0.0.0 --port 8080
 ```
 ```bash
 curl http://localhost:8080/health
 ```
 ---
 ## 2. Build and run the Docker image locally
 ```bash
 docker build -t ghaymah-api:latest ./q1-deploy-monitor/app
 ```
 ```bash
 docker run --rm -p 8080:8080 --name ghaymah-api ghaymah-api:latest
 ```
 Verify:
 ```bash
 curl -f http://localhost:8080/health && curl http://localhost:8080/metrics
 ```
 Docker's own health probe (defined by `HEALTHCHECK` in the Dockerfile) shows up
 after ~10s in `docker ps` as `(healthy)`:
 ```bash
 docker ps --filter name=ghaymah-api
 ```
 **Image design notes** (what the Dockerfile is doing and why):
 - `python:3.12-slim` — small base, no build toolchain in the final image.
 - `requirements.txt` is copied and installed **before** the application code, so
  editing `main.py` reuses the cached dependency layer instead of reinstalling
  FastAPI on every build.
 - Runs as the non-root user `appuser` (uid 10001).
 - `curl` is the only extra apt package, installed solely for the `HEALTHCHECK`;
  apt lists are removed in the same layer.
 - `EXPOSE 8080` matches the port Ghaymah is configured with below.
 ---
 ## 3. Push the image and deploy on Ghaymah
 Ghaymah deploys a container from a **public image URL**, so the image must live in
 a public registry first. Docker Hub is used here — replace `<MY_DOCKERHUB_USER>`
 with your own account name.
 ### 3.1 Push to Docker Hub
 ```bash
 docker login
 ```
 ```bash
 docker tag ghaymah-api:latest docker.io/<MY_DOCKERHUB_USER>/ghaymah-api:latest
 ```
 ```bash
 docker push docker.io/<MY_DOCKERHUB_USER>/ghaymah-api:latest
 ```
 > Building on an Apple Silicon / ARM machine? Build for the platform Ghaymah runs
 > (`linux/amd64`) or the container will fail to start:
 > ```bash
 > docker buildx build --platform linux/amd64 -t docker.io/<MY_DOCKERHUB_USER>/ghaymah-api:latest --push ./q1-deploy-monitor/app
 > ```
 Make sure the Docker Hub repository is **public** — Ghaymah pulls the image
 anonymously from the URL you paste in.
 ### 3.2 Deploy on the Ghaymah dashboard
 | Field | Value |
 |---|---|
 | Container Image URL | `docker.io/<MY_DOCKERHUB_USER>/ghaymah-api:latest` |
 | Application Name | `ghaymah-api` |
 | Port Number | `8080` (must match the `EXPOSE`d port) |
 | Public Access | **enabled** |
 | Environment Variables | *(optional)* `APP_NAME=ghaymah-api`, `APP_VERSION=1.0.0` |
 Then click **Deploy**. Once the deployment reports as running, Ghaymah assigns the
 service a public URL.
 ### 3.3 Verify the live deployment
 ```bash
 curl -f <the public URL Ghaymah assigns>/health
 ```
 Expected: HTTP 200 with `{"status":"ok","uptime_s":...,"timestamp":"..."}`.
 Also open `<the public URL Ghaymah assigns>/docs` in a browser for the OpenAPI page,
 and screenshot both the running service in the Ghaymah dashboard and the `/health`
 response for the submission.
 > **Redeploying a new version:** push a new image tag and update the Container
 > Image URL on the service. Prefer an explicit tag (e.g. `:v2` or the git SHA)
 > over `:latest` so a redeploy is unambiguous about which build is running.
 ---
 ## 4. Run the monitor
 `monitor/monitor.py` uses the **Python standard library only** — nothing to install.
 ```bash
 export APP_URL="<the public URL Ghaymah assigns>"
 python q1-deploy-monitor/monitor/monitor.py
 ```
 On Windows PowerShell:
 ```bash
 $env:APP_URL="<the public URL Ghaymah assigns>"; python q1-deploy-monitor\monitor\monitor.py
 ```
 Every 30 seconds it issues `GET $APP_URL/health` with a 5 s timeout, then
 `GET $APP_URL/metrics`, and appends one record to `monitor/data/checks.json`
 (a JSON array, created on first run):
 ```json
 {
  "ts": "2026-07-26T15:37:29.886772+00:00",
  "status": "up",
  "code": 200,
  "latency_ms": 65.3,
  "requests": 2
 }
 ```
 - Any timeout, connection error, TLS failure or non-2xx response → `"status":"down"`
  with `latency_ms: null` (and the HTTP code when the server did answer).
 - `/metrics` is best-effort: if only that call fails, the check still counts as
  **up** and `requests` is `null`.
 - After **3 consecutive failures** it prints an `ALERT:` line to stdout (once per
  outage), and a `RECOVERED:` line when the service answers again.
 Single check (useful for cron, CI or a smoke test — exits `0` if up, `1` if down):
 ```bash
 APP_URL="<the public URL Ghaymah assigns>" python q1-deploy-monitor/monitor/monitor.py --once
 ```
 Leave the loop running well before the submission so the dashboard has real history.
 **Tuning via environment variables**
 | Variable | Default | Meaning |
 |---|---|---|
 | `APP_URL` | *(required)* | base URL of the deployed app (also settable with `--url`) |
 | `INTERVAL_S` | `30` | seconds between checks |
 | `TIMEOUT_S` | `5` | per-request timeout |
 | `ALERT_AFTER` | `3` | consecutive failures before the ALERT line |
 | `DATA_FILE` | `monitor/data/checks.json` | where records are written |
 | `MAX_RECORDS` | `2880` | rolling window (24 h at one check / 30 s) |
 Records are written atomically (temp file + rename), so the dashboard never reads
 a half-written file.
 ---
 ## 5. Open the dashboard
 The page fetches `../monitor/data/checks.json`, so it must be served over HTTP —
 opening `index.html` directly from the filesystem is blocked by the browser's
 `file://` fetch restrictions (the page detects this and tells you so instead of
 failing silently).
 ```bash
 cd q1-deploy-monitor && python -m http.server 8000
 ```
 Then open <http://localhost:8000/dashboard/index.html>.
 It shows:
 - **Status badge** — green `UP` / red `DOWN` from the most recent check, with the
  HTTP code and timestamp.
 - **Total requests** — `requests_total` from the latest check.
 - **Latest latency** + the average across the stored window.
 - **Uptime %** across all stored checks.
 - **Latency line chart** — `latency_ms` over time (last 120 checks); down checks
  appear as gaps with red points.
 - **Last-updated timestamp**, auto-refreshing every 30 s.
 With no data (monitor not started yet, file missing, or an empty/corrupt array)
 it renders a "no data yet" state and an explanatory banner rather than erroring.
 To point the page at a different data file, edit the one constant at the top of
 the `<script>` block:
 ```js
 const DATA_URL = '../monitor/data/checks.json';
 ```
 ---
 ## 6. Local verification performed
 | Check | Result |
 |---|---|
 | `pip install -r requirements.txt` (pinned versions) | fastapi 0.115.6, uvicorn 0.34.0 installed cleanly |
 | `GET /health` | HTTP 200 · `{"status":"ok","uptime_s":7.797,"timestamp":"..."}` |
 | `GET /` and `GET /metrics` | valid JSON; `requests_total` increments per request |
 | `monitor.py --once` against the running app | `UP code=200 latency=45.8ms requests=5`, exit 0 |
 | `monitor.py` loop across an app shutdown | up records → down records → `ALERT` printed on the 3rd consecutive failure |
 | Dashboard against real `checks.json` | badge, tiles, chart and uptime % all rendered; no console errors |
 | Dashboard with the data file removed | "no data yet" state + banner, no crash |
 | `docker build` | **not run** — the local Docker daemon was not running; build it with the command in §2 before pushing |
@@ -0,0 +1,29 @@
 # Python
 __pycache__/
 *.py[cod]
 *.egg-info/
 .venv/
 venv/
 env/
 .pytest_cache/
 .mypy_cache/
 .ruff_cache/
 # Tooling / editors
 .git/
 .gitignore
 .github/
 .vscode/
 .idea/
 *.swp
 # OS
 .DS_Store
 Thumbs.db
 # Project files that don't belong in the image
 *.md
 Dockerfile
 .dockerignore
 .env
 *.log
@@ -0,0 +1,33 @@
 FROM python:3.12-slim
 # curl is required by the HEALTHCHECK below; everything else stays out of the image.
 RUN apt-get update \
    && apt-get install -y --no-install-recommends curl \
    && rm -rf /var/lib/apt/lists/*
 ENV PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1 \
    PIP_NO_CACHE_DIR=1 \
    PORT=8080
 WORKDIR /app
 # Dependencies first: this layer is cached and only rebuilt when
 # requirements.txt changes, not on every code edit.
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 # Application code last — the layer that changes most often.
 COPY main.py .
 # Run as an unprivileged user.
 RUN useradd --create-home --uid 10001 appuser \
    && chown -R appuser:appuser /app
 USER appuser
 EXPOSE 8080
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
    CMD curl -f http://localhost:8080/health || exit 1
 CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"]
@@ -0,0 +1,88 @@
 """Ghaymah demo API — liveness and request metrics with no external dependencies.
 Endpoints:
    GET /         basic service info
    GET /health   liveness probe (process-only, no downstream checks)
    GET /metrics  in-memory request counter
 """
 import os
 import time
 from datetime import datetime, timezone
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
 APP_NAME = os.getenv("APP_NAME", "ghaymah-api")
 APP_VERSION = os.getenv("APP_VERSION", "1.0.0")
 PORT = int(os.getenv("PORT", "8080"))
 # Process start time drives both uptime and the /metrics started_at field.
 STARTED_AT_MONOTONIC = time.monotonic()
 STARTED_AT_ISO = datetime.now(timezone.utc).isoformat()
 app = FastAPI(
    title=APP_NAME,
    version=APP_VERSION,
    description="Minimal API deployed on the Ghaymah container platform.",
 )
 # In-memory counter. Deliberately per-process: it resets on restart, which is
 # exactly what we want the dashboard to show after a redeploy or a crash-loop.
 _requests_total = 0
 def _now_iso() -> str:
    return datetime.now(timezone.utc).isoformat()
 def _uptime_s() -> float:
    return round(time.monotonic() - STARTED_AT_MONOTONIC, 3)
@app.middleware("http")
 async def count_requests(request: Request, call_next):
    """Increment the counter for every request, including failed ones."""
    global _requests_total
    _requests_total += 1
    return await call_next(request)
@app.get("/")
 async def root():
    return {
        "service": APP_NAME,
        "version": APP_VERSION,
        "message": "Ghaymah deployment demo API",
        "endpoints": ["/", "/health", "/metrics", "/docs"],
        "started_at": STARTED_AT_ISO,
        "timestamp": _now_iso(),
    }
@app.get("/health")
 async def health():
    """Liveness only — no DB or network calls, so it never fails for a
    downstream reason the orchestrator cannot fix by restarting us."""
    return JSONResponse(
        status_code=200,
        content={
            "status": "ok",
            "uptime_s": _uptime_s(),
            "timestamp": _now_iso(),
        },
    )
@app.get("/metrics")
 async def metrics():
    return {
        "requests_total": _requests_total,
        "started_at": STARTED_AT_ISO,
    }
 if __name__ == "__main__":
    import uvicorn
    uvicorn.run("main:app", host="0.0.0.0", port=PORT, log_level="info")
@@ -0,0 +1,2 @@
 fastapi==0.115.6
 uvicorn[standard]==0.34.0
@@ -0,0 +1,398 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
 <meta charset="utf-8">
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title>Ghaymah API — Monitoring Dashboard</title>
 <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
 <style>
  :root {
    --bg: #0d1117;
    --panel: #161b22;
    --panel-2: #1c2129;
    --border: #262d38;
    --text: #e6edf3;
    --muted: #8b949e;
    --up: #3fb950;
    --up-dim: rgba(63, 185, 80, .14);
    --down: #f85149;
    --down-dim: rgba(248, 81, 73, .14);
    --accent: #58a6ff;
    --warn: #d29922;
  }
  * { box-sizing: border-box; }
  body {
    margin: 0;
    padding: 32px 24px 56px;
    background: var(--bg);
    color: var(--text);
    font: 15px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto,
          "Helvetica Neue", Arial, sans-serif;
    -webkit-font-smoothing: antialiased;
  }
  .wrap { max-width: 1080px; margin: 0 auto; }
  header {
    display: flex;
    flex-wrap: wrap;
    gap: 12px;
    align-items: baseline;
    justify-content: space-between;
    margin-bottom: 28px;
  }
  h1 {
    margin: 0;
    font-size: 22px;
    font-weight: 600;
    letter-spacing: -.01em;
  }
  h1 span { color: var(--muted); font-weight: 400; }
  .updated { font-size: 13px; color: var(--muted); }
  .updated b { color: var(--text); font-weight: 500; }
  .grid {
    display: grid;
    grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
    gap: 16px;
    margin-bottom: 16px;
  }
  .card {
    background: var(--panel);
    border: 1px solid var(--border);
    border-radius: 12px;
    padding: 20px;
  }
  .card .label {
    font-size: 12px;
    text-transform: uppercase;
    letter-spacing: .08em;
    color: var(--muted);
    margin-bottom: 10px;
  }
  .card .value {
    font-size: 30px;
    font-weight: 650;
    letter-spacing: -.02em;
    font-variant-numeric: tabular-nums;
  }
  .card .sub { margin-top: 6px; font-size: 13px; color: var(--muted); }
  /* Status card */
  .status-card { display: flex; flex-direction: column; justify-content: center; }
  .badge {
    display: inline-flex;
    align-items: center;
    gap: 10px;
    padding: 10px 18px;
    border-radius: 999px;
    font-size: 22px;
    font-weight: 700;
    letter-spacing: .04em;
    align-self: flex-start;
  }
  .badge .dot {
    width: 11px; height: 11px; border-radius: 50%;
    background: currentColor;
    box-shadow: 0 0 0 0 currentColor;
  }
  .badge.up   { color: var(--up);   background: var(--up-dim);   border: 1px solid rgba(63,185,80,.35); }
  .badge.down { color: var(--down); background: var(--down-dim); border: 1px solid rgba(248,81,73,.35); }
  .badge.unknown { color: var(--muted); background: #1c2129; border: 1px solid var(--border); font-size: 18px; }
  .badge.up .dot { animation: pulse 2.2s ease-out infinite; }
  @keyframes pulse {
    0%   { box-shadow: 0 0 0 0 rgba(63,185,80,.55); }
    70%  { box-shadow: 0 0 0 9px rgba(63,185,80,0); }
    100% { box-shadow: 0 0 0 0 rgba(63,185,80,0); }
  }
  .chart-card { padding: 20px 20px 12px; }
  .chart-head {
    display: flex; align-items: baseline; justify-content: space-between;
    margin-bottom: 14px;
  }
  .chart-head .label { margin: 0; }
  .chart-box { position: relative; height: 300px; }
  .empty {
    display: flex; align-items: center; justify-content: center;
    height: 100%; min-height: 120px;
    color: var(--muted); font-size: 14px; text-align: center;
    border: 1px dashed var(--border); border-radius: 10px;
  }
  .banner {
    display: none;
    margin-bottom: 16px;
    padding: 12px 16px;
    border-radius: 10px;
    border: 1px solid rgba(210,153,34,.35);
    background: rgba(210,153,34,.12);
    color: var(--warn);
    font-size: 13.5px;
  }
  .banner.show { display: block; }
  .banner code {
    background: rgba(255,255,255,.06); padding: 1px 5px; border-radius: 4px;
    font-size: 12.5px; color: var(--text);
  }
  footer {
    margin-top: 24px; font-size: 12.5px; color: var(--muted);
    display: flex; justify-content: space-between; flex-wrap: wrap; gap: 8px;
  }
  code.path { color: var(--accent); }
 </style>
 </head>
 <body>
 <div class="wrap">
  <header>
    <h1>Ghaymah API <span>· monitoring</span></h1>
    <div class="updated">Last updated <b id="updated">—</b></div>
  </header>
  <div id="banner" class="banner"></div>
  <div class="grid">
    <div class="card status-card">
      <div class="label">Current status</div>
      <div id="badge" class="badge unknown"><span class="dot"></span><span id="badgeText">NO DATA</span></div>
      <div class="sub" id="statusSub">Waiting for the first check…</div>
    </div>
    <div class="card">
      <div class="label">Total requests</div>
      <div class="value" id="requests">—</div>
      <div class="sub" id="requestsSub">from /metrics</div>
    </div>
    <div class="card">
      <div class="label">Latest latency</div>
      <div class="value" id="latency">—</div>
      <div class="sub" id="latencySub">avg — · checks —</div>
    </div>
    <div class="card">
      <div class="label">Uptime (all checks)</div>
      <div class="value" id="uptime">—</div>
      <div class="sub" id="uptimeSub">— up / — down</div>
    </div>
  </div>
  <div class="card chart-card">
    <div class="chart-head">
      <div class="label">Response time — latency_ms over time</div>
      <div class="updated" id="chartRange"></div>
    </div>
    <div class="chart-box">
      <canvas id="chart"></canvas>
      <div id="chartEmpty" class="empty" style="display:none">No data yet — start the monitor to collect checks.</div>
    </div>
  </div>
  <footer>
    <span>Data source: <code class="path" id="srcPath"></code></span>
    <span>Auto-refresh every 30s</span>
  </footer>
 </div>
 <script>
 // ---------------------------------------------------------------------------
 // Path to the JSON array written by monitor/monitor.py. Change if you serve
 // the dashboard from a different location.
 const DATA_URL = '../monitor/data/checks.json';
 const REFRESH_MS = 30000;
 const MAX_POINTS = 120; // most recent N checks shown on the chart
 // ---------------------------------------------------------------------------
 const $ = (id) => document.getElementById(id);
 $('srcPath').textContent = DATA_URL;
 let chart = null;
 function fmtTime(iso) {
  const d = new Date(iso);
  if (isNaN(d)) return String(iso ?? '—');
  return d.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' });
 }
 function fmtDateTime(iso) {
  const d = new Date(iso);
  if (isNaN(d)) return String(iso ?? '—');
  return d.toLocaleString([], { month: 'short', day: 'numeric',
    hour: '2-digit', minute: '2-digit', second: '2-digit' });
 }
 function showBanner(msg) {
  const b = $('banner');
  b.innerHTML = msg;
  b.classList.add('show');
 }
 function hideBanner() { $('banner').classList.remove('show'); }
 function setBadge(state, text) {
  const badge = $('badge');
  badge.className = 'badge ' + state;
  $('badgeText').textContent = text;
 }
 function renderEmpty(reason) {
  setBadge('unknown', 'NO DATA');
  $('statusSub').textContent = reason || 'Waiting for the first check…';
  $('requests').textContent = '—';
  $('latency').textContent = '—';
  $('latencySub').textContent = 'avg — · checks —';
  $('uptime').textContent = '—';
  $('uptimeSub').textContent = '— up / — down';
  $('chartRange').textContent = '';
  $('chartEmpty').style.display = 'flex';
  $('chart').style.display = 'none';
  if (chart) { chart.destroy(); chart = null; }
 }
 function renderChart(points) {
  $('chartEmpty').style.display = 'none';
  $('chart').style.display = 'block';
  const labels = points.map(p => fmtTime(p.ts));
  // Down checks have no latency — null creates a visible gap in the line.
  const data = points.map(p => (typeof p.latency_ms === 'number' ? p.latency_ms : null));
  const colors = points.map(p => (p.status === 'up' ? '#3fb950' : '#f85149'));
  if (chart) {
    chart.data.labels = labels;
    chart.data.datasets[0].data = data;
    chart.data.datasets[0].pointBackgroundColor = colors;
    chart.update('none');
    return;
  }
  const ctx = $('chart').getContext('2d');
  const gradient = ctx.createLinearGradient(0, 0, 0, 300);
  gradient.addColorStop(0, 'rgba(88,166,255,.28)');
  gradient.addColorStop(1, 'rgba(88,166,255,0)');
  chart = new Chart(ctx, {
    type: 'line',
    data: {
      labels,
      datasets: [{
        label: 'latency (ms)',
        data,
        borderColor: '#58a6ff',
        backgroundColor: gradient,
        borderWidth: 2,
        pointRadius: 2.5,
        pointHoverRadius: 5,
        pointBackgroundColor: colors,
        pointBorderWidth: 0,
        tension: .3,
        fill: true,
        spanGaps: false,
      }]
    },
    options: {
      responsive: true,
      maintainAspectRatio: false,
      animation: false,
      interaction: { mode: 'index', intersect: false },
      plugins: {
        legend: { display: false },
        tooltip: {
          backgroundColor: '#1c2129',
          borderColor: '#262d38',
          borderWidth: 1,
          titleColor: '#e6edf3',
          bodyColor: '#8b949e',
          padding: 10,
          displayColors: false,
          callbacks: {
            label: (c) => c.parsed.y === null ? 'down — no response' : `${c.parsed.y.toFixed(1)} ms`
          }
        }
      },
      scales: {
        x: {
          grid: { color: 'rgba(255,255,255,.05)' },
          ticks: { color: '#8b949e', maxTicksLimit: 10, maxRotation: 0, autoSkip: true }
        },
        y: {
          beginAtZero: true,
          grid: { color: 'rgba(255,255,255,.05)' },
          ticks: { color: '#8b949e', callback: (v) => v + ' ms' }
        }
      }
    }
  });
 }
 function render(records) {
  if (!Array.isArray(records) || records.length === 0) {
    renderEmpty('No checks recorded yet.');
    return;
  }
  const valid = records.filter(r => r && typeof r === 'object');
  if (valid.length === 0) { renderEmpty('Data file contains no usable checks.'); return; }
  const latest = valid[valid.length - 1];
  const isUp = latest.status === 'up';
  setBadge(isUp ? 'up' : 'down', isUp ? 'UP' : 'DOWN');
  $('statusSub').textContent =
    (latest.code != null ? `HTTP ${latest.code}` : 'no response') + ' · ' + fmtDateTime(latest.ts);
  $('requests').textContent =
    typeof latest.requests === 'number' ? latest.requests.toLocaleString() : '—';
  $('requestsSub').textContent =
    typeof latest.requests === 'number' ? 'from /metrics' : '/metrics unavailable';
  $('latency').textContent =
    typeof latest.latency_ms === 'number' ? `${latest.latency_ms.toFixed(0)} ms` : '—';
  const latencies = valid.map(r => r.latency_ms).filter(v => typeof v === 'number');
  const avg = latencies.length
    ? (latencies.reduce((a, b) => a + b, 0) / latencies.length).toFixed(0) + ' ms'
    : '—';
  $('latencySub').textContent = `avg ${avg} · ${valid.length} checks`;
  const upCount = valid.filter(r => r.status === 'up').length;
  const pct = (upCount / valid.length) * 100;
  $('uptime').textContent = `${pct.toFixed(1)}%`;
  $('uptimeSub').textContent = `${upCount} up / ${valid.length - upCount} down`;
  const points = valid.slice(-MAX_POINTS);
  $('chartRange').textContent =
    points.length > 1 ? `${fmtTime(points[0].ts)} → ${fmtTime(points[points.length - 1].ts)}` : '';
  renderChart(points);
 }
 async function load() {
  try {
    const res = await fetch(`${DATA_URL}?t=${Date.now()}`, { cache: 'no-store' });
    if (!res.ok) throw new Error(`HTTP ${res.status}`);
    const data = await res.json();
    hideBanner();
    render(data);
  } catch (err) {
    // Missing file, invalid JSON, or file:// CORS block — never crash the page.
    renderEmpty('Could not load monitoring data.');
    showBanner(
      `Could not read <code>${DATA_URL}</code> (${err.message}). ` +
      `Run the monitor to create it, and serve this page over HTTP ` +
      `(<code>python -m http.server 8000</code> from <code>q1-deploy-monitor/</code>) ` +
      `— opening the file directly with <code>file://</code> blocks the fetch.`
    );
  }
  $('updated').textContent = new Date().toLocaleTimeString();
 }
 load();
 setInterval(load, REFRESH_MS);
 </script>
 </body>
 </html>
@@ -0,0 +1,625 @@
 [
  {
    "ts": "2026-07-26T16:00:54.022563+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 48.44,
    "requests": 5
  },
  {
    "ts": "2026-07-26T16:01:24.088718+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 12.02,
    "requests": 8
  },
  {
    "ts": "2026-07-26T16:01:54.237407+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 24.42,
    "requests": 11
  },
  {
    "ts": "2026-07-26T16:02:24.326513+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 10.23,
    "requests": 14
  },
  {
    "ts": "2026-07-26T16:02:54.359082+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.32,
    "requests": 19
  },
  {
    "ts": "2026-07-26T16:03:24.389873+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 7.83,
    "requests": 22
  },
  {
    "ts": "2026-07-26T16:03:54.430952+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 11.57,
    "requests": 25
  },
  {
    "ts": "2026-07-26T16:04:24.463587+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.01,
    "requests": 28
  },
  {
    "ts": "2026-07-26T16:04:54.502315+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 11.91,
    "requests": 31
  },
  {
    "ts": "2026-07-26T16:05:24.545377+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 7.72,
    "requests": 34
  },
  {
    "ts": "2026-07-26T16:05:54.596001+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.86,
    "requests": 37
  },
  {
    "ts": "2026-07-26T16:06:24.623029+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.53,
    "requests": 40
  },
  {
    "ts": "2026-07-26T16:06:54.659187+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.9,
    "requests": 43
  },
  {
    "ts": "2026-07-26T16:07:24.697242+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 11.24,
    "requests": 46
  },
  {
    "ts": "2026-07-26T16:07:54.732633+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.15,
    "requests": 49
  },
  {
    "ts": "2026-07-26T16:08:24.764311+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 7.73,
    "requests": 52
  },
  {
    "ts": "2026-07-26T16:08:54.794053+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 7.53,
    "requests": 55
  },
  {
    "ts": "2026-07-26T16:09:24.830576+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 10.88,
    "requests": 58
  },
  {
    "ts": "2026-07-26T16:09:55.392240+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 9.02,
    "requests": 61
  },
  {
    "ts": "2026-07-26T16:10:25.432620+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 7.75,
    "requests": 64
  },
  {
    "ts": "2026-07-26T16:10:55.469947+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 9.34,
    "requests": 67
  },
  {
    "ts": "2026-07-26T16:11:25.502913+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 7.92,
    "requests": 70
  },
  {
    "ts": "2026-07-26T16:11:55.542608+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 11.26,
    "requests": 73
  },
  {
    "ts": "2026-07-26T16:12:25.579631+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 10.91,
    "requests": 76
  },
  {
    "ts": "2026-07-26T16:12:55.621565+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.02,
    "requests": 79
  },
  {
    "ts": "2026-07-26T16:13:25.651738+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 7.58,
    "requests": 82
  },
  {
    "ts": "2026-07-26T16:13:55.683853+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 9.19,
    "requests": 85
  },
  {
    "ts": "2026-07-26T16:14:25.720913+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 10.67,
    "requests": 88
  },
  {
    "ts": "2026-07-26T16:14:55.754438+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.68,
    "requests": 91
  },
  {
    "ts": "2026-07-26T16:15:25.795149+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 9.94,
    "requests": 94
  },
  {
    "ts": "2026-07-26T16:15:55.835606+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.67,
    "requests": 97
  },
  {
    "ts": "2026-07-26T16:16:25.894224+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 21.72,
    "requests": 100
  },
  {
    "ts": "2026-07-26T16:16:55.947740+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.81,
    "requests": 103
  },
  {
    "ts": "2026-07-26T16:17:25.993670+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 14.76,
    "requests": 106
  },
  {
    "ts": "2026-07-26T16:17:56.027955+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 9.95,
    "requests": 109
  },
  {
    "ts": "2026-07-26T16:18:26.076150+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 16.53,
    "requests": 112
  },
  {
    "ts": "2026-07-26T16:18:56.130454+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 13.91,
    "requests": 115
  },
  {
    "ts": "2026-07-26T16:19:26.181118+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 14.55,
    "requests": 118
  },
  {
    "ts": "2026-07-26T16:19:56.239466+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 15.54,
    "requests": 121
  },
  {
    "ts": "2026-07-26T16:20:26.273327+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 12.98,
    "requests": 124
  },
  {
    "ts": "2026-07-26T16:20:56.326056+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 10.54,
    "requests": 127
  },
  {
    "ts": "2026-07-26T16:21:26.360754+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.28,
    "requests": 130
  },
  {
    "ts": "2026-07-26T16:21:56.419699+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 24.33,
    "requests": 133
  },
  {
    "ts": "2026-07-26T16:22:26.474809+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 16.0,
    "requests": 136
  },
  {
    "ts": "2026-07-26T16:22:56.537278+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 25.59,
    "requests": 139
  },
  {
    "ts": "2026-07-26T16:23:26.588750+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 11.74,
    "requests": 142
  },
  {
    "ts": "2026-07-26T16:23:56.625763+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 11.76,
    "requests": 145
  },
  {
    "ts": "2026-07-26T16:24:26.678666+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 21.03,
    "requests": 148
  },
  {
    "ts": "2026-07-26T16:24:56.718058+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 9.81,
    "requests": 151
  },
  {
    "ts": "2026-07-26T16:25:26.781135+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 26.99,
    "requests": 154
  },
  {
    "ts": "2026-07-26T16:25:56.837517+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 17.22,
    "requests": 157
  },
  {
    "ts": "2026-07-26T16:26:26.885310+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 15.99,
    "requests": 160
  },
  {
    "ts": "2026-07-26T16:26:56.931050+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 12.71,
    "requests": 163
  },
  {
    "ts": "2026-07-26T16:27:26.978657+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 15.71,
    "requests": 166
  },
  {
    "ts": "2026-07-26T16:27:57.027222+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 15.87,
    "requests": 169
  },
  {
    "ts": "2026-07-26T16:28:27.076324+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 11.98,
    "requests": 172
  },
  {
    "ts": "2026-07-26T16:28:57.144076+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 24.72,
    "requests": 175
  },
  {
    "ts": "2026-07-26T16:29:27.203425+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 20.56,
    "requests": 178
  },
  {
    "ts": "2026-07-26T16:29:57.243755+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 7.91,
    "requests": 181
  },
  {
    "ts": "2026-07-26T16:30:27.305659+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 19.38,
    "requests": 184
  },
  {
    "ts": "2026-07-26T16:30:57.380343+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 37.99,
    "requests": 187
  },
  {
    "ts": "2026-07-26T16:31:27.412862+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.02,
    "requests": 190
  },
  {
    "ts": "2026-07-26T16:31:57.458486+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 16.76,
    "requests": 193
  },
  {
    "ts": "2026-07-26T16:32:27.506190+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 10.54,
    "requests": 196
  },
  {
    "ts": "2026-07-26T16:32:57.558316+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 19.22,
    "requests": 199
  },
  {
    "ts": "2026-07-26T16:33:27.636370+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 18.47,
    "requests": 202
  },
  {
    "ts": "2026-07-26T16:33:57.692190+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 20.24,
    "requests": 205
  },
  {
    "ts": "2026-07-26T16:34:27.737228+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 10.93,
    "requests": 208
  },
  {
    "ts": "2026-07-26T16:34:57.800304+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 22.88,
    "requests": 211
  },
  {
    "ts": "2026-07-26T16:35:27.861991+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 18.19,
    "requests": 214
  },
  {
    "ts": "2026-07-26T16:35:57.925683+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 19.93,
    "requests": 217
  },
  {
    "ts": "2026-07-26T16:36:27.992496+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 18.07,
    "requests": 220
  },
  {
    "ts": "2026-07-26T16:36:58.067518+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 18.62,
    "requests": 223
  },
  {
    "ts": "2026-07-26T16:37:28.127809+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 20.59,
    "requests": 226
  },
  {
    "ts": "2026-07-26T16:37:58.166210+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 9.57,
    "requests": 229
  },
  {
    "ts": "2026-07-26T16:38:28.201786+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 7.95,
    "requests": 232
  },
  {
    "ts": "2026-07-26T16:38:58.237767+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.12,
    "requests": 235
  },
  {
    "ts": "2026-07-26T16:39:28.279617+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 11.27,
    "requests": 238
  },
  {
    "ts": "2026-07-26T16:39:58.317037+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 7.61,
    "requests": 241
  },
  {
    "ts": "2026-07-26T16:40:28.355043+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 9.57,
    "requests": 244
  },
  {
    "ts": "2026-07-26T16:40:58.391288+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 9.27,
    "requests": 247
  },
  {
    "ts": "2026-07-26T16:41:28.432715+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 13.3,
    "requests": 250
  },
  {
    "ts": "2026-07-26T16:41:58.503792+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 26.98,
    "requests": 253
  },
  {
    "ts": "2026-07-26T16:42:28.608440+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 19.86,
    "requests": 256
  },
  {
    "ts": "2026-07-26T16:42:58.693278+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 12.8,
    "requests": 259
  },
  {
    "ts": "2026-07-26T16:43:28.797562+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 42.28,
    "requests": 262
  },
  {
    "ts": "2026-07-26T16:43:58.874830+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 8.97,
    "requests": 265
  },
  {
    "ts": "2026-07-26T16:44:28.919903+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 13.56,
    "requests": 268
  },
  {
    "ts": "2026-07-26T16:44:59.037760+00:00",
    "status": "up",
    "code": 200,
    "latency_ms": 34.65,
    "requests": 271
  }
 ]
@@ -0,0 +1,204 @@
 #!/usr/bin/env python3
 """Uptime monitor for the API deployed on Ghaymah.
 Polls ``$APP_URL/health`` (and ``/metrics``) every 30 seconds and appends one
 record per check to ``monitor/data/checks.json`` — the file the dashboard reads.
 Record schema:
    {"ts": "<iso8601>", "status": "up"|"down", "code": <int|null>,
     "latency_ms": <float|null>, "requests": <int|null>}
 Usage:
    APP_URL=https://my-app.example  python monitor.py
    APP_URL=https://my-app.example  python monitor.py --once
 Standard library only — no pip install needed to run the monitor.
 """
 import argparse
 import json
 import os
 import sys
 import time
 import urllib.error
 import urllib.request
 from datetime import datetime, timezone
 from pathlib import Path
 # --- Configuration (env-overridable) -----------------------------------------
 APP_URL = os.getenv("APP_URL", "").rstrip("/")
 INTERVAL_S = float(os.getenv("INTERVAL_S", "30"))
 TIMEOUT_S = float(os.getenv("TIMEOUT_S", "5"))
 ALERT_AFTER = int(os.getenv("ALERT_AFTER", "3"))
 DEFAULT_DATA_FILE = Path(__file__).resolve().parent / "data" / "checks.json"
 DATA_FILE = Path(os.getenv("DATA_FILE", str(DEFAULT_DATA_FILE)))
 # Keep the JSON array bounded so the dashboard stays fast and the file small.
 MAX_RECORDS = int(os.getenv("MAX_RECORDS", "2880"))  # 24h at one check / 30s
 def now_iso() -> str:
    return datetime.now(timezone.utc).isoformat()
 def fetch_json(url: str, timeout: float):
    """GET a URL and parse JSON. Returns (status_code, parsed_body).
    Raises on timeout, connection failure, or non-2xx status.
    """
    req = urllib.request.Request(url, headers={"User-Agent": "ghaymah-monitor/1.0"})
    with urllib.request.urlopen(req, timeout=timeout) as resp:
        raw = resp.read().decode("utf-8", errors="replace")
        try:
            body = json.loads(raw)
        except json.JSONDecodeError:
            body = None
        return resp.status, body
 def check_once(app_url: str) -> dict:
    """Run one health + metrics check and return the record to persist."""
    health_url = f"{app_url}/health"
    metrics_url = f"{app_url}/metrics"
    started = time.perf_counter()
    try:
        code, _body = fetch_json(health_url, TIMEOUT_S)
        latency_ms = round((time.perf_counter() - started) * 1000, 2)
        status = "up"
    except urllib.error.HTTPError as exc:
        # The app answered, just not with a healthy status — record the code.
        return {
            "ts": now_iso(),
            "status": "down",
            "code": exc.code,
            "latency_ms": round((time.perf_counter() - started) * 1000, 2),
            "requests": None,
        }
    except Exception:
        # Timeout, DNS failure, connection refused, TLS error, ...
        return {
            "ts": now_iso(),
            "status": "down",
            "code": None,
            "latency_ms": None,
            "requests": None,
        }
    # /metrics is best-effort: a failure there must not turn a healthy app "down".
    requests_total = None
    try:
        _mcode, mbody = fetch_json(metrics_url, TIMEOUT_S)
        if isinstance(mbody, dict):
            value = mbody.get("requests_total")
            if isinstance(value, int):
                requests_total = value
    except Exception:
        pass
    return {
        "ts": now_iso(),
        "status": status,
        "code": code,
        "latency_ms": latency_ms,
        "requests": requests_total,
    }
 def load_records(path: Path) -> list:
    """Read the existing JSON array; tolerate a missing or corrupt file."""
    if not path.exists():
        return []
    try:
        with path.open("r", encoding="utf-8") as fh:
            data = json.load(fh)
        return data if isinstance(data, list) else []
    except (json.JSONDecodeError, OSError):
        print(f"[warn] {path} unreadable or corrupt — starting a fresh array", file=sys.stderr)
        return []
 def append_record(path: Path, record: dict) -> None:
    """Append one record to the JSON array, writing atomically."""
    path.parent.mkdir(parents=True, exist_ok=True)
    records = load_records(path)
    records.append(record)
    if len(records) > MAX_RECORDS:
        records = records[-MAX_RECORDS:]
    tmp = path.with_suffix(path.suffix + ".tmp")
    with tmp.open("w", encoding="utf-8") as fh:
        json.dump(records, fh, indent=2)
        fh.write("\n")
    tmp.replace(path)
 def format_line(record: dict) -> str:
    latency = f"{record['latency_ms']:.1f}ms" if record["latency_ms"] is not None else "  -  "
    code = record["code"] if record["code"] is not None else "---"
    reqs = record["requests"] if record["requests"] is not None else "-"
    return (
        f"{record['ts']}  {record['status'].upper():<4}  "
        f"code={code:<4} latency={latency:<9} requests={reqs}"
    )
 def run(app_url: str, once: bool) -> int:
    consecutive_failures = 0
    alerted = False
    while True:
        record = check_once(app_url)
        append_record(DATA_FILE, record)
        print(format_line(record), flush=True)
        if record["status"] == "down":
            consecutive_failures += 1
            if consecutive_failures >= ALERT_AFTER and not alerted:
                print(
                    f"ALERT: {app_url} has failed {consecutive_failures} consecutive "
                    f"health checks (since {record['ts']})",
                    flush=True,
                )
                alerted = True
        else:
            if alerted:
                print(f"RECOVERED: {app_url} is responding again at {record['ts']}", flush=True)
            consecutive_failures = 0
            alerted = False
        if once:
            return 0 if record["status"] == "up" else 1
        time.sleep(INTERVAL_S)
 def main() -> int:
    parser = argparse.ArgumentParser(description="Monitor the Ghaymah-deployed API.")
    parser.add_argument("--once", action="store_true", help="run a single check and exit")
    parser.add_argument("--url", default=APP_URL, help="app base URL (default: $APP_URL)")
    args = parser.parse_args()
    app_url = (args.url or "").rstrip("/")
    if not app_url:
        print(
            "error: no app URL. Set APP_URL, e.g.\n"
            "  APP_URL=https://my-app.ghaymah.example python monitor.py",
            file=sys.stderr,
        )
        return 2
    print(f"Monitoring {app_url} every {INTERVAL_S:g}s (timeout {TIMEOUT_S:g}s)", flush=True)
    print(f"Writing checks to {DATA_FILE}", flush=True)
    try:
        return run(app_url, args.once)
    except KeyboardInterrupt:
        print("\nStopped.", flush=True)
        return 0
 if __name__ == "__main__":
    sys.exit(main())
@@ -0,0 +1,134 @@
 # Postmortem: Repeated OOMKilled Container Restarts
 **Severity:** SEV-2  
 **Incident date:** 2026-07-24  
 **Duration:** 45 minutes (14:05–14:50 EEST)  
 **Status:** Resolved
 ## 1. Summary
 On 2026-07-24, the application was unavailable for 45 minutes after its containers entered a repeated `OOMKilled` crash loop with exit code 137. Memory usage had grown gradually following a release that introduced an unbounded in-process cache; once each container reached its configured memory limit, the container runtime terminated it. Automated client retries and queued traffic increased load on the briefly recovering containers, accelerating subsequent failures. Users experienced an estimated 95–100% request error rate during the incident, amounting to effective full downtime. The incident was classified as **SEV-2** because the production service was unavailable, but no data loss or security impact was identified.
 ## 2. Timeline
 All times are EEST (UTC+3).
 | Time | Event | Actor/action |
 |---|---|---|
 | 13:40 | Release `v2.18.0` is deployed. It contains a new in-process response cache. | Deployment pipeline completes successfully; smoke checks pass. |
 | 13:45 | Memory begins rising above the previous baseline of approximately 55% per container. | Application traffic populates the cache; no alert exists for sustained memory growth. |
 | 13:52 | Memory reaches approximately 75% and continues to grow without returning to baseline. | No action is taken because memory-usage alerting is not configured. |
 | 14:03 | One container approaches its configured memory limit; garbage collection activity and latency increase. | Container continues serving requests with degraded response times. |
 | 14:05 | First container is terminated by the runtime as `OOMKilled` with exit code 137. | Runtime restart policy starts a replacement container; incident begins (minute 0). |
 | 14:07 | The replacement becomes ready, receives queued and retried requests, and its cache grows rapidly. | Client retries and redistributed traffic increase pressure on the remaining containers. |
 | 14:10 | Multiple containers enter a restart loop; error rate exceeds 95%. | Runtime repeatedly restarts terminated containers, but they fail again after becoming ready. |
 | 14:13 | Availability alert fires after sustained health-check failures. | On-call engineer is paged and acknowledges the alert. |
 | 14:16 | On-call confirms widespread HTTP 5xx/timeouts and repeated container restarts. | Engineer begins incident triage and declares SEV-2. |
 | 14:20 | Exit code 137 and `OOMKilled` events are identified; memory graphs show steady growth after the release. | Engineer correlates the failure pattern with deployment `v2.18.0`. |
 | 14:24 | Incident lead pauses nonessential changes and selects rollback as the primary mitigation. | A second engineer reviews the previous stable image and rollback procedure. |
 | 14:28 | Memory limits are temporarily raised to provide recovery headroom while rollback proceeds. | Platform operator updates the workload configuration and replaces affected containers. |
 | 14:33 | Rollback to `v2.17.4` starts. | Deployment pipeline replaces the leaking release with the last known-good image. |
 | 14:38 | First rolled-back containers become healthy; error rate begins to fall. | On-call watches health checks, memory, restarts, latency, and request errors. |
 | 14:42 | All active containers run `v2.17.4`; memory stabilizes near the prior baseline. | Traffic is allowed to normalize; retry pressure subsides. |
 | 14:47 | Health checks remain successful and no new OOM kills occur for five minutes. | Incident lead begins the recovery validation period. |
 | 14:50 | Error rate and latency return to normal; full service is confirmed. | Incident lead resolves the incident after 45 minutes of effective downtime. |
 ## 3. Root Cause
 ### 5-Whys analysis
 1. **Why was the application unavailable?**  
   Its production containers repeatedly terminated and restarted, leaving insufficient healthy capacity to serve requests.
 2. **Why did the containers terminate?**  
   Each container exceeded its memory limit and was killed by the container runtime, producing an `OOMKilled` event and exit code 137.
 3. **Why did memory exceed the limit?**  
   Release `v2.18.0` introduced an unbounded in-process response cache. Cache entries were not evicted, so memory grew continuously with request diversity.
 4. **Why did the release cause a crash loop rather than degrade safely?**  
   The memory limit was sized for the old application baseline and did not include sufficient headroom for the new allocation pattern. Restarted instances were also immediately exposed to queued and retried traffic, causing the cache to refill quickly.
 5. **Why was the defect not detected before or shortly after deployment?**  
   CI did not include memory-regression soak or load testing, and production monitoring did not alert on sustained memory utilization, abnormal restart count, or `OOMKilled` events.
 **Root cause:** A memory leak introduced in `v2.18.0`—an unbounded in-process cache—caused container memory to grow until it exceeded a limit sized for the previous baseline. The impact was compounded by retry traffic, insufficient memory headroom, absent memory and restart alerting, and the lack of memory-regression testing in CI.
 ## 4. Recommendations
 | Action | Owner (role) | Priority |
 |---|---|---|
 | Replace the unbounded cache with a size- and TTL-bounded implementation; add eviction metrics and tests. | Application engineering lead | P0 |
 | Recalculate container memory requests/limits from measured peak usage and retain operational headroom. | Platform/SRE engineer | P0 |
 | Alert when per-container memory utilization exceeds 80% for five minutes. | Observability/SRE engineer | P0 |
 | Alert when a workload records more than three restarts in ten minutes, and surface `OOMKilled`/exit-code-137 events. | Observability/SRE engineer | P0 |
 | Implement the proposed horizontal auto-scaling policy below after load validation. | Platform/SRE engineer | P1 |
 | Add sustained soak/load tests and memory-regression thresholds to CI before production promotion. | Quality engineering and application teams | P1 |
 | Write and exercise an OOM/crash-loop response runbook covering diagnosis, rollback, temporary headroom, and retry control. | Incident management/SRE lead | P1 |
 | Add bounded exponential backoff and jitter to applicable client and internal retries. | Application engineering team | P2 |
 ## 5. Proposed Auto-scaling Policy for Ghaymah
 This is a proposed policy design for workloads deployed on Ghaymah; it does not assume a named or currently documented Ghaymah autoscaling feature.
 ```yaml
 policy_name: production-api-resource-scaling
 mode: horizontal
 replicas:
  minimum: 2
  maximum: 10  # N; validate against load tests, quota, and downstream capacity
 scale_out:
  evaluation_window: 2m
  conditions:
    operator: OR
    rules:
      - metric: average_container_memory_utilization
        threshold: "> 70%"
      - metric: average_container_cpu_utilization
        threshold: "> 65%"
  increment:
    strategy: proportional
    minimum_replicas: 1
    maximum_step_percent: 50
 scale_in:
  memory_threshold: "< 45%"
  cpu_threshold: "< 40%"
  evaluation_window: 10m
  condition_operator: AND
  decrement: 1
  cooldown: 10m
 stabilization:
  scale_out_cooldown: 2m
  scale_in_cooldown: 10m
 ```
 The maximum `N` is initially **10 replicas** and must be validated against measured container throughput, account capacity, and the limits of databases and other downstream services. Scale-out occurs when either average memory exceeds 70% or average CPU exceeds 65% for two continuous minutes. Scale-in requires both metrics to remain low and uses a 10-minute cooldown—within the requested 5–10 minute range—to prevent flapping. At least two replicas remain active for availability.
 Autoscaling alone does **not** fix a memory leak: every new replica runs the same defective code and will eventually leak. Horizontal scaling buys response time, preserves capacity during a gradual rise, and absorbs the retry storm while engineers mitigate the defect. It should be complemented by a properly measured memory limit with vertical headroom, bounded retries, readiness checks, and a restart policy that replaces failed containers without creating an uncontrolled hot loop. The lasting corrective action is to remove the leak.
 ## 6. Early Detection with a Monitoring Approach on Ghaymah
 Because Ghaymah's public documentation does not establish specific monitoring, alerting, or autoscaling product features, the following approach uses platform-agnostic container signals. These signals can be collected from container runtime statistics and events, combined with an external monitor that calls the application's `/health` endpoint.
 ### Signals and patterns
 - Plot per-container memory working set and memory utilization against the configured limit. A steady creep after a deployment, or a repeating sawtooth pattern that rises to the limit and drops when a container restarts, is an early indicator of a leak or crash loop.
 - Track container restart count and restart rate by workload, release version, and container instance.
 - Capture termination reason, especially `OOMKilled`, and exit code 137.
 - Monitor `/health` availability and latency externally so an alert remains effective even when the application or container-level telemetry is unavailable.
 - Correlate memory, restarts, HTTP error rate, latency, and deployments on the same operational dashboard.
 ### Alert rules
 | Signal | Proposed rule | Purpose |
 |---|---|---|
 | Memory utilization | Per-container memory > 80% of limit for 5 minutes | Warn before the runtime enforces the limit. |
 | Restart count | More than 3 restarts for the same workload in 10 minutes | Detect a crash loop early. |
 | OOM event | Any `OOMKilled` event or exit code 137 in production | Page immediately on confirmed memory exhaustion. |
 | Health availability | Two or more consecutive `/health` failures from an external monitor | Detect loss of service independently of container telemetry. |
 | Memory growth | Sustained positive memory slope after a release, with no return toward baseline | Identify slow leaks before the hard threshold is reached. |
 Operational dashboards should show the current release version alongside memory utilization, limit, restart count, termination reason, health status, latency, and error rate. Alerts should route to the on-call notification channel with the workload, container, release, current memory percentage, restart count, and a link to the OOM runbook.
 <!-- VERIFY: Confirm whether Ghaymah provides native container runtime metrics/events, configurable dashboards, alert rules, and notification-channel integrations. If not, export runtime signals to an external monitoring system and use an external /health probe. -->
@@ -0,0 +1,138 @@
 # CI/CD Pipeline for Ghaymah
 ## Pipeline overview
 The workflow builds the API from `q1-deploy-monitor/app`, publishes two Docker Hub tags, deploys the immutable commit tag to a separate staging application, pauses for manual approval, and then deploys the exact same image to production. Promoting the same immutable `${{ github.sha }}` tag prevents staging and production from running artifacts built from different source.
 ```mermaid
 flowchart LR
    A["Commit to main"] --> B["Build image"]
    B --> C["Push SHA and latest tags<br/>to Docker Hub"]
    C --> D["Deploy staging<br/>(automatic)"]
    D --> E{"Manual approval<br/>GitHub Environment"}
    E -->|Approved| F["Deploy production"]
    E -->|Rejected| G["Stop deployment"]
 ```
 The workflow runs on:
 - A push to `main`.
 - A manual `workflow_dispatch` run.
 It uses these GitHub Actions secrets:
 | Secret | Purpose |
 |---|---|
 | `DOCKERHUB_USERNAME` | Docker Hub account/namespace used to authenticate and construct the image name. |
 | `DOCKERHUB_TOKEN` | Docker Hub access token used to push images. |
 | `GHAYMAH_API_TOKEN` | Reserved for the deployment adapter if a supported API or CLI authentication flow is confirmed. |
 Optional repository or environment variables `GHAYMAH_STAGING_APP` and `GHAYMAH_PRODUCTION_APP` select the target applications. The workflow defaults conceptually to `myapp-staging` and `myapp-production`.
 Images are published as:
 ```text
 docker.io/<user>/ghaymah-api:<git-commit-sha>
 docker.io/<user>/ghaymah-api:latest
 ```
 The SHA tag is used for deployment because it is immutable and auditable. `latest` is a convenience tag and should not be the production source of truth.
 Ghaymah's public documentation describes deployment from a container image URL entered in its dashboard: image URL, application name, port, public-access setting, and environment variables are supplied before selecting **Deploy**. A Ghaymah-hosted registry is not assumed. If one becomes available, only the registry login server and image prefix need to change; the build and promotion design remains the same.
 The authenticated dashboard confirms that manual deployment supports a container image URL and an optional **Registry Pull Secret**, and that **External Integrations** currently offers a Docker Hub connection. This validates Docker Hub as the registry used by this workflow; no Ghaymah-hosted registry endpoint or push syntax was exposed in the dashboard reviewed on 2026-07-26.
 ## Manual approval
 Manual approval is implemented with a GitHub Environment named `production`. The workflow declares:
 ```yaml
 environment: production
 ```
 Required reviewers are configured in the GitHub repository UI, not in workflow YAML:
 1. Open **Settings → Environments**.
 2. Create or select the `production` environment.
 3. Enable the deployment protection rule for required reviewers.
 4. Add the people or teams authorized to approve production deployments.
 5. Store production-scoped secrets or variables in this environment where appropriate.
 After staging succeeds, the `deploy-production` job enters a waiting state. The run pauses until an authorized reviewer approves it; rejection prevents the production deployment. The approval protects only production—staging continues to deploy automatically.
 ## Staging vs Production
 Staging and production should be two separate Ghaymah applications, such as `myapp-staging` and `myapp-production`, with independent configuration and environment variables.
 | Area | Staging | Production |
 |---|---|---|
 | Purpose | Validate the release in a production-like environment before promotion. | Serve the live customer workload. |
 | Data | Synthetic, anonymized, or disposable test data. | Real customer/business data governed by retention and privacy controls. |
 | Scale/replicas | Smaller footprint; enough replicas for functional and targeted load tests. | Sized for peak traffic, resilience, and operational headroom. |
 | Secrets | Staging-only credentials with limited permissions. | Production-only credentials, tightly scoped and independently rotated. |
 | Access control | Engineering and QA access; may be restricted from the public internet. | Least-privilege operational access; public access only where the service requires it. |
 | Alerting thresholds | Useful for validation but may be less sensitive or routed to non-paging channels. | SLO-based thresholds with paging for user-impacting failures. |
 | Deploy cadence | Automatic after each successful build from `main`. | Only after staging succeeds and a reviewer approves the deployment. |
 | Who can approve | No approval required for this pipeline. | Reviewers assigned to the GitHub `production` Environment. |
 Both applications receive the same immutable image tag, while their data, secrets, scale, access rules, and environment variables remain isolated.
 ## Ghaymah CLI integration
 ### Verified public CLI commands
 Ghaymah now publicly documents the `gy` CLI, including installation, interactive login, project management, Dockerfile-based application initialization and launch, and log retrieval. The currently documented flow is:
 ```bash
 curl -sSL https://cli.ghaymah.systems/install.sh | bash
 source ~/.bashrc
 gy version
 gy auth login
 gy auth status
 gy resource project get
 gy resource app init --project-id <PROJECT_ID> --name <APP_NAME>
 gy resource app launch [PATH]
 gy resource app logs
 gy resource app update <APP_ID> --set '<CONFIRMED_UPDATE_FIELDS>'
 ```
 `app init` creates `.ghaymah.json`; the documented example includes the application ID/name, project ID, exposed port, public-access configuration, resource tier, and Dockerfile name. `app launch` builds the local Dockerfile and deploys that application.
 Official sources:
 - [Ghaymah CLI documentation](https://ghaymah.systems/docs)
 - [Ghaymah CLI overview](https://ghaymah.systems/cli)
 ### CI integration gap
 Direct inspection of CLI version `0.0.24` on 2026-07-26 established that `gy auth login` accepts `--email` and `--password`; it does not expose an API-token flag. It also established that `gy resource app update <APP_ID>` accepts JSON input or dot notation through `--set`.
 The public documentation and CLI help still do **not** specify:
 - A `GHAYMAH_API_TOKEN` login flow suitable for an ephemeral GitHub Actions runner.
 - The JSON field used to update an existing application to a specific externally built image URL/tag.
 - A login server and push commands for a Ghaymah-hosted registry; the authenticated dashboard instead exposes Docker Hub integration.
 <!-- VERIFY: Confirm whether Ghaymah supports API-token authentication and identify the external-image field accepted by `gy resource app update`. -->
 Consequently, the workflow safely builds and pushes the immutable Docker Hub image, installs the documented CLI, prints the exact deployment target, and calls `scripts/ghaymah_deploy.sh`. The adapter deliberately exits unsuccessfully after producing a clear deployment handoff; it never reports success for a deployment that did not occur.
 Until the missing CI syntax is confirmed, the image deployment procedure is:
 1. Open the target Ghaymah application in the dashboard.
 2. Update its container image URL to `docker.io/<user>/ghaymah-api:<git-commit-sha>`.
 3. Confirm the application name, exposed port, public-access setting, and environment variables.
 4. Select **Deploy** and validate application health.
 5. Repeat for production only after the GitHub Environment approval.
 The staging and production workflow steps print the exact SHA-tagged image and target application, then call:
 <!-- VERIFY: Replace this fail-safe adapter with confirmed non-interactive authentication and external-image update syntax. -->
 ```bash
 bash scripts/ghaymah_deploy.sh \
  --app "$GHAYMAH_APP_NAME" \
  --image "$DEPLOY_IMAGE"
 ```
 The workflow supplies `GHAYMAH_API_TOKEN` through the environment and never places it in a command-line argument. Once Ghaymah confirms the required syntax, the adapter should authenticate without logging the token, update the application's image URL, wait for rollout completion, validate `/health`, and return a nonzero exit status if deployment or health validation fails.
@@ -0,0 +1,114 @@
 # Scalability and Load Distribution on Ghaymah
 ## 1. Architecture for 15,000 requests per second
 ```mermaid
 flowchart LR
    U["Clients<br/>15,000 req/s"] -->|"Resolve application hostname"| DNS["DNS"]
    DNS -->|"Route requests"| CDN["CDN / edge cache<br/>absorbs cacheable bursts"]
    CDN -->|"Cache misses and dynamic requests"| LB["L7 load balancer<br/>TLS termination and routing"]
    subgraph API["Auto-scaled stateless API fleet"]
        direction TB
        C1["API containers<br/>minimum capacity: 39"]
        C2["Additional containers<br/>during scale-out / deploys"]
    end
    LB -->|"Distribute dynamic traffic"| C1
    LB -->|"Sharp-load scale-out"| C2
    C1 -->|"Hot-key and session-independent reads"| REDIS["Redis cache<br/>absorbs repeated-read bursts"]
    C2 -->|"Hot-key and session-independent reads"| REDIS
    C1 -->|"Writes and cache misses"| PRIMARY["PostgreSQL primary<br/>Ghaymah Block Storage volume"]
    C2 -->|"Writes and cache misses"| PRIMARY
    REDIS -->|"Cache miss"| PRIMARY
    PRIMARY -->|"Streaming replication"| R1["PostgreSQL read replica 1<br/>Ghaymah Block Storage volume"]
    PRIMARY -->|"Streaming replication"| R2["PostgreSQL read replica 2<br/>Ghaymah Block Storage volume"]
    C1 -->|"Read-only queries"| R1
    C2 -->|"Read-only queries"| R2
    DNS -.-> MON["Monitoring and alerting<br/>metrics, logs, traces, health checks"]
    CDN -.-> MON
    LB -.-> MON
    C1 -.-> MON
    C2 -.-> MON
    REDIS -.-> MON
    PRIMARY -.-> MON
    R1 -.-> MON
    R2 -.-> MON
 ```
 The normal request path is **DNS → CDN/edge cache → L7 load balancer → stateless API container**. Cacheable content is answered at the CDN edge, so it does not consume API capacity. Dynamic requests reach the container fleet, where Redis absorbs repeated reads and hot-key bursts. Database writes go to the PostgreSQL primary, while eligible read-only traffic is distributed across read replicas. The monitoring component observes health, throughput, saturation, latency, errors, logs, and traces across every tier.
 The CDN and Redis cache are the first two burst buffers. They reduce the amount of work reaching the API and database, but capacity planning still assumes the API may need to handle the full 15,000 req/s. This conservative assumption prevents cache misses, invalidation events, or an unusually write-heavy workload from immediately exhausting the fleet.
 ## 2. Container count calculation
 Given:
 - Peak application load: **15,000 req/s**
 - Tested capacity per container: **500 req/s**
 - Required headroom: **30%**
 Base container count:
 ```text
 15,000 req/s ÷ 500 req/s per container = 30 containers
 ```
 Add 30% headroom:
 ```text
 30 × 1.30 = 39 containers
 ```
 **Answer: 39 application containers.**
 Capacity calculations must always round fractional results up to the next whole container. The fleet should also permit at least **N+1**, or **40 active container slots**, during rolling deployments so one extra container can become ready before an old container is removed. The 30% operating headroom is reserved to absorb traffic spikes and the modeled loss of a zone without exceeding the tested 500 req/s-per-container limit. The zone distribution and failure model must be load-tested to confirm that 30% is sufficient for the actual placement topology.
 The 500 req/s figure must represent a sustainable result at acceptable latency and error rate—not a short-lived maximum from a synthetic test. Database, Redis, network, and load-balancer capacity must be tested at the same target because adding API containers cannot remove a downstream bottleneck.
 ## 3. Cold-start strategy
 - **Maintain a warm replica floor.** Keep at least the calculated normal-capacity fleet available for this high-throughput tier and never scale it to zero. Retain additional warm capacity when traffic is volatile or startup time is longer than the acceptable scaling delay.
 - **Use slim, pre-pulled images.** Build from a small production base image, exclude build tools and development dependencies, and keep the layer count low. Pre-pull the approved image onto worker capacity where the runtime permits it so a scale-out does not wait for a large registry download.
 - **Make application boot fast.** Initialize only request-critical dependencies during startup. Lazy-load optional modules, background reports, large reference datasets, and other non-critical work after the process can safely serve traffic.
 - **Enforce a readiness probe.** The load balancer must route traffic only after the process is listening and critical dependencies are usable. Readiness should remain false while caches, connection pools, or mandatory configuration are warming; liveness should be separate so slow startup is not mistaken for a dead process.
 - **Scale predictively before known peaks.** Use historical demand and scheduled events to add capacity several minutes before daily peaks, campaigns, batch workloads, or announced launches. Reactive scaling then handles deviations from the forecast.
 - **Use step-based scale-out.** Under a sharp rise, add several containers in one scaling action rather than one at a time. Choose step sizes from measured startup duration and traffic growth—for example, add 25% of the current fleet when utilization crosses the high threshold, subject to the maximum capacity and downstream limits.
 - **Warm new instances progressively.** Ramp traffic to newly ready containers rather than sending a full share immediately. This allows connection pools and caches to warm without creating a synchronized load spike against Redis or PostgreSQL.
 - **Control retries during startup.** Apply bounded exponential backoff with jitter and retry budgets so failed requests do not multiply the load while new capacity is becoming ready.
 ## 4. Ghaymah Block Storage for stateful workloads
 Containers are ephemeral: their writable container filesystem may disappear when an instance is restarted, replaced, rescheduled, or scaled in. Durable state must therefore live outside the application container. Applied to Ghaymah, a block-storage volume should be attached to a stateful service and formatted or managed by that service just as a persistent disk would be.
 Appropriate uses include:
 - PostgreSQL database files and write-ahead logs.
 - Durable queue or broker data when the selected queue requires a filesystem-backed store.
 - User uploads when a filesystem interface is required, although object storage is generally preferable for horizontally shared upload data.
 The stateless API tier remains completely diskless. It stores no durable sessions, uploads, or business records on its local container filesystem, so any API instance can be created, replaced, or removed without data migration.
 ### Attachment and scaling model
 A conventional filesystem on a block volume should be treated as a **single-writer resource** unless the storage service and filesystem explicitly support safe multi-writer attachment. Multiple API or database containers must not concurrently mount and write to the same ordinary filesystem volume. The stateful database tier scales through database-aware replication:
 1. The PostgreSQL primary owns its data volume and accepts writes.
 2. Each read replica owns a separate volume containing its replicated database copy.
 3. PostgreSQL replication transfers changes from the primary to the replicas.
 4. Read traffic can be distributed to replicas; writes continue to use the primary.
 This design does not clone and concurrently mount one writable volume across replicas. Replication preserves database consistency at the application layer and allows each database instance to manage its own disk.
 ### Backup, performance, and capacity
 Crash-consistent or application-consistent volume snapshots can provide a backup building block, but database-aware backups and restore tests are still required. A snapshot should be coordinated with PostgreSQL or combined with its write-ahead log archive so the recovery point is valid.
 <!-- VERIFY: Confirm that Ghaymah Block Storage supports snapshots, including snapshot consistency behavior, retention, encryption, restore procedure, and cross-region or cross-zone availability. -->
 The authenticated Ghaymah dashboard displays a supported volume-size range of **50 MiB minimum to 10 GiB maximum** and allows a volume to be attached from an application's advanced deployment options. Volume sizing must account for the live dataset, indexes, temporary files, write-ahead logs, maintenance operations, expected growth, and free-space safety margin. Performance planning must account for sustained and burst IOPS, throughput, latency, queue depth, and the read/write mix. Load tests should verify database latency at expected peak traffic rather than selecting capacity from size alone.
 <!-- VERIFY: Confirm resize behavior, supported volume classes, IOPS/throughput limits, attachment limits, access modes, and zone-binding rules before final production sizing. -->
 Block Storage makes container replacement compatible with durable state, but it does not replace database replication, tested backups, point-in-time recovery, or a documented failover procedure.
@@ -0,0 +1,41 @@
 FROM python:3.12-slim
 # curl is used by the HEALTHCHECK below.
 RUN apt-get update \
    && apt-get install -y --no-install-recommends curl \
    && rm -rf /var/lib/apt/lists/*
 ENV PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1 \
    PIP_NO_CACHE_DIR=1 \
    PORT=8080 \
    WEB_ROOT=/app/dashboard \
    METRICS_FILE=/app/dashboard/data/metrics.json
 WORKDIR /app
 # Dependencies first so editing the collector or dashboard doesn't reinstall them.
 COPY collector/requirements.txt ./collector/requirements.txt
 RUN pip install --no-cache-dir -r collector/requirements.txt
 # Application: the collector and the static dashboard it feeds.
 COPY collector/collect.py ./collector/collect.py
 COPY dashboard/index.html ./dashboard/index.html
 COPY start.sh ./start.sh
 # Tolerate CRLF line endings if the repo was checked out on Windows.
 RUN sed -i 's/\r$//' ./start.sh && chmod +x ./start.sh
 # Non-root user; it must be able to write metrics into the served directory.
 RUN useradd --create-home --uid 10001 appuser \
    && mkdir -p /app/dashboard/data \
    && chown -R appuser:appuser /app
 USER appuser
 EXPOSE 8080
 # The dashboard page is the liveness signal for the whole container.
 HEALTHCHECK --interval=60s --timeout=5s --start-period=15s --retries=3 \
    CMD curl -f http://localhost:8080/index.html || exit 1
 CMD ["./start.sh"]
@@ -0,0 +1,211 @@
 # Q5 — Monitoring dashboard for mithal.space
 A single container that continuously measures the availability and performance of
 **https://mithal.space** and serves a live dashboard of the results on port 8080.
 ```
 q5-mithal-dashboard/
 ├── collector/
 │   ├── collect.py             # measures latency, uptime, DNS, TLS, search — every 60s
 │   └── requirements.txt       # requests (pinned); everything else is stdlib
 ├── dashboard/
 │   ├── index.html             # self-contained dashboard (Chart.js from CDN)
 │   └── data/metrics.json      # rolling 48h JSON array, written by the collector
 ├── start.sh                   # entrypoint: collector in background + static server
 ├── Dockerfile
 └── README.md
 ```
 **Why `data/` lives inside `dashboard/`:** the dashboard directory *is* the web
 root, so the page fetches `data/metrics.json` from its own origin. One container,
 one port, no CORS, no API layer.
 ---
 ## 1. What is collected
 Every 60 seconds `collector/collect.py` appends one record to the JSON array:
 ```json
 {
  "ts": "2026-07-26T15:46:40.381676+00:00",
  "up": true,
  "code": 200,
  "latency_ms": 848.58,
  "dns_ms": 0.67,
  "ssl_days_left": 50,
  "search_ms": 1056.08
 }
 ```
 | Field | How it is measured |
 |---|---|
 | `latency_ms` | timed `GET https://mithal.space`, 10 s timeout, redirects followed |
 | `up` | `true` when the status code is **200–399** |
 | `code` | the HTTP status code — `null` when the connection itself failed |
 | `dns_ms` | timed `socket.getaddrinfo("mithal.space", 443)` |
 | `ssl_days_left` | TLS handshake to `mithal.space:443`, cert `notAfter` parsed → days remaining |
 | `search_ms` | timed `GET https://mithal.space/search?q=test` (`null` if it errors or returns ≥ 400) |
 Every measurement is independent: a failure records `null` for that field only and
 never aborts the run or crashes the loop. Records older than **48 hours** are
 pruned on each write, and the file is written atomically (temp file + rename) so
 the dashboard never reads a half-written array.
 **Configuration** — constants at the top of `collect.py`, all overridable by env var:
 | Variable | Default | Meaning |
 |---|---|---|
 | `TARGET_URL` | `https://mithal.space` | site under test |
 | `SEARCH_URL` | `https://mithal.space/search?q=test` | search endpoint to time |
 | `INTERVAL_S` | `60` | seconds between collections |
 | `TIMEOUT_S` | `10` | per-request timeout |
 | `RETENTION_HOURS` | `48` | how much history to keep |
 | `METRICS_FILE` | `dashboard/data/metrics.json` | output path |
 ---
 ## 2. Run locally (without Docker)
 ```bash
 cd q5-mithal-dashboard
 python -m venv .venv && source .venv/bin/activate   # Windows: .venv\Scripts\activate
 pip install -r collector/requirements.txt
 ```
 One-shot collection (useful as a smoke test or under cron):
 ```bash
 python collector/collect.py --once
 ```
 Continuous collection every 60 s — leave this running:
 ```bash
 python collector/collect.py
 ```
 In a second terminal, serve the dashboard (the `data/` directory must be inside
 the served root, which it is):
 ```bash
 cd q5-mithal-dashboard/dashboard && python -m http.server 8080
 ```
 Open <http://localhost:8080/index.html>.
 > Opening `index.html` straight from disk does **not** work — browsers block
 > `fetch()` over `file://`. The page detects this and shows an explanatory banner
 > instead of failing silently.
 ---
 ## 3. The dashboard
 | Element | Detail |
 |---|---|
 | Status badge | green `UP` / red `DOWN` from the newest record, with HTTP code and timestamp |
 | Uptime tile | `up_checks / total_checks × 100` over the **last 24 h**, one decimal |
 | TLS card | "*X* days remaining" — green > 30, yellow 8–30, red ≤ 7 (`Expired` at ≤ 0) |
 | Latest response | newest `latency_ms`, with `search_ms` and `dns_ms` underneath |
 | Chart | `latency_ms` (solid blue) and `search_ms` (dashed purple) for the **last hour**; failed checks draw gaps, red points mark down checks |
 | Table | last 10 checks — time, ✅/❌, code, latency, DNS, search |
 | Refresh | re-fetches every 60 s; last-updated clock in the header |
 Empty, missing, or corrupt data renders a "no data yet" state on every tile plus a
 banner explaining what to do — it never throws.
 To point the page elsewhere, edit the one constant at the top of the `<script>`:
 ```js
 const DATA_URL = 'data/metrics.json';
 ```
 ---
 ## 4. Build the image
 The container runs the collector in the background and serves the dashboard in the
 foreground, both from one process tree (`start.sh`):
 ```bash
 docker build -t mithal-monitor:latest ./q5-mithal-dashboard
 ```
 ```bash
 docker run --rm -p 8080:8080 --name mithal-monitor mithal-monitor:latest
 ```
 Open <http://localhost:8080/index.html>. The first record appears within a few
 seconds of startup, then one per minute.
 Image notes:
 - `python:3.12-slim`, non-root user `appuser` (uid 10001) owning the served tree
  so the collector can write into it.
 - `requirements.txt` installed before the code is copied, for layer caching.
 - `start.sh` restarts the collector if it ever exits, and traps `SIGTERM`/`SIGINT`
  so the container stops promptly.
 - `HEALTHCHECK` fetches the dashboard page itself.
 - Metrics live inside the container's filesystem, so **history resets on redeploy**.
  That is intentional for this exercise; to keep history across restarts, mount a
  volume at `/app/dashboard/data` (this is exactly what Ghaymah Block Storage is
  for — see Q4).
 ---
 ## 5. Deploy to Ghaymah
 Ghaymah deploys from a **public image URL**, so push the image to Docker Hub first.
 Replace `<MY_DOCKERHUB_USER>` with your account name.
 ```bash
 docker login
 ```
 ```bash
 docker tag mithal-monitor:latest docker.io/<MY_DOCKERHUB_USER>/mithal-monitor:latest
 ```
 ```bash
 docker push docker.io/<MY_DOCKERHUB_USER>/mithal-monitor:latest
 ```
 > On Apple Silicon / ARM, build for the platform Ghaymah runs:
 > ```bash
 > docker buildx build --platform linux/amd64 -t docker.io/<MY_DOCKERHUB_USER>/mithal-monitor:latest --push ./q5-mithal-dashboard
 > ```
 Make sure the Docker Hub repository is **public** — Ghaymah pulls it anonymously.
 Then, in the Ghaymah dashboard:
 | Field | Value |
 |---|---|
 | Container Image URL | `docker.io/<MY_DOCKERHUB_USER>/mithal-monitor:latest` |
 | Application Name | `mithal-monitor` |
 | Port Number | `8080` (matches `EXPOSE`) |
 | Public Access | **enabled** |
 | Environment Variables | *(optional)* `INTERVAL_S=60`, `TARGET_URL=https://mithal.space`, `SEARCH_URL=https://mithal.space/search?q=test` |
 Click **Deploy**, then open `<the public URL Ghaymah assigns>/index.html` and
 screenshot the live dashboard. Leave it running so the 24 h uptime tile and the
 hourly chart fill with real history before submission.
 ---
 ## 6. Local verification performed
 Measured against the live site on 2026-07-26:
 | Check | Result |
 |---|---|
 | `collect.py --once` against mithal.space | `UP code=200 latency=2693ms dns=7.07ms search=866ms ssl=50d` |
 | 60 s loop (run at 6 s for testing) | 5 further records appended, pruning and atomic writes working |
 | Failure path (unreachable host) | recorded `up:false` with `code/latency_ms/dns_ms/ssl_days_left/search_ms` all `null`, exit 0, no crash |
 | Dashboard against real `metrics.json` | badge UP, uptime 100.0% (6/6), TLS card "50 days — Healthy" in green, chart with both series, 6-row table; no console errors |
 | Dashboard with the data file removed | "no data yet" state on every tile + banner, no crash |
 | `docker build` | **not run** — the local Docker daemon was not running; build with the command in §4 before pushing |
 The `dashboard/data/metrics.json` in this repo holds those first real samples; the
 collector prunes anything older than 48 h automatically.
@@ -0,0 +1,237 @@
 #!/usr/bin/env python3
 """Metrics collector for https://mithal.space.
 Every 60 seconds (or once with --once) it measures availability, page latency,
 DNS resolution time, TLS certificate lifetime and search-endpoint latency, then
 appends one record to the JSON array the dashboard reads.
 Record schema:
    {"ts": "<iso8601 utc>", "up": bool, "code": int|null, "latency_ms": float|null,
     "dns_ms": float|null, "ssl_days_left": int|null, "search_ms": float|null}
 A failing measurement never aborts the run — it is recorded as null.
 Usage:
    python collect.py            # loop every 60s
    python collect.py --once     # single collection, then exit
 """
 import argparse
 import json
 import os
 import socket
 import ssl
 import sys
 import time
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from urllib.parse import urlparse
 import requests
 # --- Configuration -----------------------------------------------------------
 TARGET_URL = os.getenv("TARGET_URL", "https://mithal.space")
 # Confirmed working search endpoint for mithal.space.
 SEARCH_URL = os.getenv("SEARCH_URL", "https://mithal.space/search?q=test")
 INTERVAL_S = float(os.getenv("INTERVAL_S", "60"))
 TIMEOUT_S = float(os.getenv("TIMEOUT_S", "10"))
 RETENTION_HOURS = float(os.getenv("RETENTION_HOURS", "48"))
 # The dashboard is served as the web root and fetches "data/metrics.json",
 # so the data directory lives inside dashboard/ both locally and in the image.
 DEFAULT_METRICS_FILE = (
    Path(__file__).resolve().parent.parent / "dashboard" / "data" / "metrics.json"
 )
 METRICS_FILE = Path(os.getenv("METRICS_FILE", str(DEFAULT_METRICS_FILE)))
 USER_AGENT = "mithal-monitor/1.0 (+uptime collector)"
 def now_utc() -> datetime:
    return datetime.now(timezone.utc)
 def host_port(url: str) -> tuple:
    """Extract (hostname, port) from a URL, defaulting to 443 for https."""
    parsed = urlparse(url)
    port = parsed.port or (443 if parsed.scheme == "https" else 80)
    return parsed.hostname, port
 # --- Individual measurements (each returns None on failure) ------------------
 def measure_page(url: str):
    """Timed GET of the home page. Returns (code|None, latency_ms|None)."""
    started = time.perf_counter()
    try:
        resp = requests.get(
            url, timeout=TIMEOUT_S, headers={"User-Agent": USER_AGENT}, allow_redirects=True
        )
        latency_ms = round((time.perf_counter() - started) * 1000, 2)
        return resp.status_code, latency_ms
    except requests.RequestException:
        # Connection refused, DNS failure, TLS error, timeout — no status code.
        return None, None
 def measure_search(url: str):
    """Timed GET of the search endpoint. Returns search_ms|None."""
    started = time.perf_counter()
    try:
        resp = requests.get(
            url, timeout=TIMEOUT_S, headers={"User-Agent": USER_AGENT}, allow_redirects=True
        )
        if resp.status_code >= 400:
            return None
        return round((time.perf_counter() - started) * 1000, 2)
    except requests.RequestException:
        return None
 def measure_dns(hostname: str, port: int):
    """Timed DNS resolution. Returns dns_ms|None."""
    started = time.perf_counter()
    try:
        socket.getaddrinfo(hostname, port)
        return round((time.perf_counter() - started) * 1000, 2)
    except (socket.gaierror, OSError):
        return None
 def measure_ssl_days_left(hostname: str, port: int):
    """Days until the TLS certificate expires. Returns int|None."""
    context = ssl.create_default_context()
    try:
        with socket.create_connection((hostname, port), timeout=TIMEOUT_S) as sock:
            with context.wrap_socket(sock, server_hostname=hostname) as tls:
                cert = tls.getpeercert()
    except (ssl.SSLError, socket.error, OSError):
        return None
    not_after = (cert or {}).get("notAfter")
    if not not_after:
        return None
    try:
        # OpenSSL format, always in GMT: "Sep 12 23:59:59 2026 GMT"
        expires = datetime.strptime(not_after, "%b %d %H:%M:%S %Y %Z").replace(
            tzinfo=timezone.utc
        )
    except ValueError:
        return None
    return (expires - now_utc()).days
 def collect_once() -> dict:
    """Run every measurement and build one record. Never raises."""
    hostname, port = host_port(TARGET_URL)
    dns_ms = measure_dns(hostname, port) if hostname else None
    code, latency_ms = measure_page(TARGET_URL)
    ssl_days_left = measure_ssl_days_left(hostname, port) if hostname else None
    search_ms = measure_search(SEARCH_URL)
    return {
        "ts": now_utc().isoformat(),
        "up": code is not None and 200 <= code < 400,
        "code": code,
        "latency_ms": latency_ms,
        "dns_ms": dns_ms,
        "ssl_days_left": ssl_days_left,
        "search_ms": search_ms,
    }
 # --- Storage -----------------------------------------------------------------
 def load_records(path: Path) -> list:
    """Read the existing JSON array; tolerate a missing or corrupt file."""
    if not path.exists():
        return []
    try:
        with path.open("r", encoding="utf-8") as fh:
            data = json.load(fh)
        return data if isinstance(data, list) else []
    except (json.JSONDecodeError, OSError):
        print(f"[warn] {path} unreadable or corrupt — starting a fresh array", file=sys.stderr)
        return []
 def prune(records: list) -> list:
    """Drop records older than the retention window."""
    cutoff = now_utc() - timedelta(hours=RETENTION_HOURS)
    kept = []
    for rec in records:
        if not isinstance(rec, dict):
            continue
        try:
            ts = datetime.fromisoformat(str(rec.get("ts")))
        except (TypeError, ValueError):
            continue
        if ts.tzinfo is None:
            ts = ts.replace(tzinfo=timezone.utc)
        if ts >= cutoff:
            kept.append(rec)
    return kept
 def append_record(path: Path, record: dict) -> None:
    """Append a record, prune the window, and write atomically."""
    path.parent.mkdir(parents=True, exist_ok=True)
    records = prune(load_records(path))
    records.append(record)
    tmp = path.with_suffix(path.suffix + ".tmp")
    with tmp.open("w", encoding="utf-8") as fh:
        json.dump(records, fh, indent=2)
        fh.write("\n")
    tmp.replace(path)
 def format_line(rec: dict) -> str:
    def num(value, suffix=""):
        return f"{value}{suffix}" if value is not None else "-"
    return (
        f"{rec['ts']}  {'UP  ' if rec['up'] else 'DOWN'}  "
        f"code={num(rec['code']):<4} "
        f"latency={num(rec['latency_ms'], 'ms'):<10} "
        f"dns={num(rec['dns_ms'], 'ms'):<9} "
        f"search={num(rec['search_ms'], 'ms'):<10} "
        f"ssl={num(rec['ssl_days_left'], 'd')}"
    )
 def main() -> int:
    parser = argparse.ArgumentParser(description="Collect availability metrics for mithal.space.")
    parser.add_argument("--once", action="store_true", help="collect a single sample and exit")
    args = parser.parse_args()
    print(f"Target:  {TARGET_URL}", flush=True)
    print(f"Search:  {SEARCH_URL}", flush=True)
    print(f"Output:  {METRICS_FILE} (keeping {RETENTION_HOURS:g}h)", flush=True)
    if not args.once:
        print(f"Interval: {INTERVAL_S:g}s", flush=True)
    while True:
        try:
            record = collect_once()
            append_record(METRICS_FILE, record)
            print(format_line(record), flush=True)
        except Exception as exc:  # a bug here must not kill a long-running collector
            print(f"[error] collection failed: {exc!r}", file=sys.stderr, flush=True)
        if args.once:
            return 0
        time.sleep(INTERVAL_S)
 if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        print("\nStopped.", flush=True)
@@ -0,0 +1 @@
 requests==2.32.3
@@ -0,0 +1,56 @@
 [
  {
    "ts": "2026-07-26T15:44:13.109453+00:00",
    "up": true,
    "code": 200,
    "latency_ms": 2693.14,
    "dns_ms": 7.07,
    "ssl_days_left": 50,
    "search_ms": 866.47
  },
  {
    "ts": "2026-07-26T15:46:03.954555+00:00",
    "up": true,
    "code": 200,
    "latency_ms": 1057.54,
    "dns_ms": 7.42,
    "ssl_days_left": 50,
    "search_ms": 958.25
  },
  {
    "ts": "2026-07-26T15:46:12.414865+00:00",
    "up": true,
    "code": 200,
    "latency_ms": 1016.96,
    "dns_ms": 0.56,
    "ssl_days_left": 50,
    "search_ms": 942.73
  },
  {
    "ts": "2026-07-26T15:46:20.678443+00:00",
    "up": true,
    "code": 200,
    "latency_ms": 781.13,
    "dns_ms": 0.38,
    "ssl_days_left": 50,
    "search_ms": 956.94
  },
  {
    "ts": "2026-07-26T15:46:32.017215+00:00",
    "up": true,
    "code": 200,
    "latency_ms": 1828.99,
    "dns_ms": 0.5,
    "ssl_days_left": 50,
    "search_ms": 2967.25
  },
  {
    "ts": "2026-07-26T15:46:40.381676+00:00",
    "up": true,
    "code": 200,
    "latency_ms": 848.58,
    "dns_ms": 0.67,
    "ssl_days_left": 50,
    "search_ms": 1056.08
  }
 ]
@@ -0,0 +1,473 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
 <meta charset="utf-8">
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title>mithal.space — Monitoring Dashboard</title>
 <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
 <style>
  :root {
    --bg: #0d1117;
    --panel: #161b22;
    --border: #262d38;
    --text: #e6edf3;
    --muted: #8b949e;
    --up: #3fb950;
    --up-dim: rgba(63,185,80,.14);
    --down: #f85149;
    --down-dim: rgba(248,81,73,.14);
    --warn: #d29922;
    --warn-dim: rgba(210,153,34,.14);
    --accent: #58a6ff;
    --accent-2: #bc8cff;
  }
  * { box-sizing: border-box; }
  body {
    margin: 0;
    padding: 32px 24px 56px;
    background: var(--bg);
    color: var(--text);
    font: 15px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto,
          "Helvetica Neue", Arial, sans-serif;
    -webkit-font-smoothing: antialiased;
  }
  .wrap { max-width: 1120px; margin: 0 auto; }
  header {
    display: flex; flex-wrap: wrap; gap: 12px;
    align-items: baseline; justify-content: space-between;
    margin-bottom: 26px;
  }
  h1 { margin: 0; font-size: 22px; font-weight: 600; letter-spacing: -.01em; }
  h1 a { color: inherit; text-decoration: none; border-bottom: 1px solid var(--border); }
  h1 a:hover { border-bottom-color: var(--accent); }
  h1 span { color: var(--muted); font-weight: 400; }
  .updated { font-size: 13px; color: var(--muted); }
  .updated b { color: var(--text); font-weight: 500; }
  .grid {
    display: grid;
    grid-template-columns: repeat(auto-fit, minmax(230px, 1fr));
    gap: 16px;
    margin-bottom: 16px;
  }
  .card {
    background: var(--panel);
    border: 1px solid var(--border);
    border-radius: 12px;
    padding: 20px;
  }
  .label {
    font-size: 12px; text-transform: uppercase; letter-spacing: .08em;
    color: var(--muted); margin-bottom: 10px;
  }
  .value {
    font-size: 30px; font-weight: 650; letter-spacing: -.02em;
    font-variant-numeric: tabular-nums;
  }
  .sub { margin-top: 6px; font-size: 13px; color: var(--muted); }
  .badge {
    display: inline-flex; align-items: center; gap: 9px;
    padding: 8px 16px; border-radius: 999px;
    font-size: 19px; font-weight: 700; letter-spacing: .04em;
    align-self: flex-start;
  }
  .badge .dot { width: 10px; height: 10px; border-radius: 50%; background: currentColor; }
  .badge.up   { color: var(--up);   background: var(--up-dim);   border: 1px solid rgba(63,185,80,.35); }
  .badge.down { color: var(--down); background: var(--down-dim); border: 1px solid rgba(248,81,73,.35); }
  .badge.unknown { color: var(--muted); background: #1c2129; border: 1px solid var(--border); font-size: 16px; }
  .badge.up .dot { animation: pulse 2.2s ease-out infinite; }
  @keyframes pulse {
    0%   { box-shadow: 0 0 0 0 rgba(63,185,80,.55); }
    70%  { box-shadow: 0 0 0 9px rgba(63,185,80,0); }
    100% { box-shadow: 0 0 0 0 rgba(63,185,80,0); }
  }
  /* SSL card colour states */
  .ssl-ok   .value { color: var(--up); }
  .ssl-warn .value { color: var(--warn); }
  .ssl-crit .value { color: var(--down); }
  .ssl-pill {
    display: inline-block; margin-top: 8px; padding: 3px 10px;
    border-radius: 999px; font-size: 12px; font-weight: 600;
  }
  .ssl-ok   .ssl-pill { color: var(--up);   background: var(--up-dim); }
  .ssl-warn .ssl-pill { color: var(--warn); background: var(--warn-dim); }
  .ssl-crit .ssl-pill { color: var(--down); background: var(--down-dim); }
  .chart-card { padding: 20px 20px 12px; }
  .chart-head {
    display: flex; align-items: baseline; justify-content: space-between;
    gap: 12px; flex-wrap: wrap; margin-bottom: 6px;
  }
  .chart-head .label { margin: 0; }
  .legend { display: flex; gap: 16px; font-size: 12.5px; color: var(--muted); margin-bottom: 12px; }
  .legend i { display: inline-block; width: 10px; height: 10px; border-radius: 2px; margin-right: 6px; }
  .chart-box { position: relative; height: 300px; }
  table { width: 100%; border-collapse: collapse; font-size: 14px; }
  th, td {
    text-align: left; padding: 10px 12px;
    border-bottom: 1px solid var(--border);
    font-variant-numeric: tabular-nums; white-space: nowrap;
  }
  th {
    font-size: 11.5px; text-transform: uppercase; letter-spacing: .07em;
    color: var(--muted); font-weight: 600;
  }
  tbody tr:last-child td { border-bottom: none; }
  tbody tr:hover { background: rgba(255,255,255,.02); }
  td.num { color: var(--text); }
  td.null { color: var(--muted); }
  .code-ok   { color: var(--up); }
  .code-bad  { color: var(--down); }
  .empty {
    display: flex; align-items: center; justify-content: center;
    height: 100%; min-height: 120px; padding: 24px;
    color: var(--muted); font-size: 14px; text-align: center;
    border: 1px dashed var(--border); border-radius: 10px;
  }
  .banner {
    display: none; margin-bottom: 16px; padding: 12px 16px;
    border-radius: 10px; border: 1px solid rgba(210,153,34,.35);
    background: rgba(210,153,34,.12); color: var(--warn); font-size: 13.5px;
  }
  .banner.show { display: block; }
  .banner code, footer code {
    background: rgba(255,255,255,.06); padding: 1px 5px; border-radius: 4px;
    font-size: 12.5px; color: var(--text);
  }
  footer {
    margin-top: 24px; font-size: 12.5px; color: var(--muted);
    display: flex; justify-content: space-between; flex-wrap: wrap; gap: 8px;
  }
  footer code.path { color: var(--accent); background: none; padding: 0; }
 </style>
 </head>
 <body>
 <div class="wrap">
  <header>
    <h1><a href="https://mithal.space" target="_blank" rel="noopener">mithal.space</a> <span>· monitoring</span></h1>
    <div class="updated">Last updated <b id="updated">—</b></div>
  </header>
  <div id="banner" class="banner"></div>
  <div class="grid">
    <div class="card">
      <div class="label">Current status</div>
      <div id="badge" class="badge unknown"><span class="dot"></span><span id="badgeText">NO DATA</span></div>
      <div class="sub" id="statusSub">Waiting for the first collection…</div>
    </div>
    <div class="card">
      <div class="label">Uptime — last 24h</div>
      <div class="value" id="uptime">—</div>
      <div class="sub" id="uptimeSub">— up / — checks</div>
    </div>
    <div class="card" id="sslCard">
      <div class="label">TLS certificate</div>
      <div class="value" id="ssl">—</div>
      <div class="sub">days remaining</div>
      <div class="ssl-pill" id="sslPill" style="display:none"></div>
    </div>
    <div class="card">
      <div class="label">Latest response</div>
      <div class="value" id="latency">—</div>
      <div class="sub" id="latencySub">search — · dns —</div>
    </div>
  </div>
  <div class="card chart-card">
    <div class="chart-head">
      <div class="label">Response time — last hour</div>
      <div class="updated" id="chartRange"></div>
    </div>
    <div class="legend">
      <span><i style="background:#58a6ff"></i>page latency</span>
      <span><i style="background:#bc8cff"></i>search latency</span>
    </div>
    <div class="chart-box">
      <canvas id="chart"></canvas>
      <div id="chartEmpty" class="empty" style="display:none">No checks in the last hour.</div>
    </div>
  </div>
  <div class="card" style="margin-top:16px">
    <div class="label">Last 10 checks</div>
    <div style="overflow-x:auto">
      <table>
        <thead>
          <tr>
            <th>Time</th><th>Status</th><th>Code</th>
            <th>Latency</th><th>DNS</th><th>Search</th>
          </tr>
        </thead>
        <tbody id="rows"></tbody>
      </table>
    </div>
    <div id="tableEmpty" class="empty" style="display:none">No checks recorded yet.</div>
  </div>
  <footer>
    <span>Data source: <code class="path" id="srcPath"></code></span>
    <span>Auto-refresh every 60s · 48h retention</span>
  </footer>
 </div>
 <script>
 // ---------------------------------------------------------------------------
 // Path to the JSON array written by collector/collect.py.
 const DATA_URL = 'data/metrics.json';
 const REFRESH_MS = 60000;
 const CHART_WINDOW_MS = 60 * 60 * 1000;   // chart shows the last hour
 const UPTIME_WINDOW_MS = 24 * 60 * 60 * 1000; // uptime % over the last 24h
 // ---------------------------------------------------------------------------
 const $ = (id) => document.getElementById(id);
 $('srcPath').textContent = DATA_URL;
 let chart = null;
 const isNum = (v) => typeof v === 'number' && isFinite(v);
 function fmtTime(iso) {
  const d = new Date(iso);
  return isNaN(d) ? String(iso ?? '—')
    : d.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' });
 }
 function fmtDateTime(iso) {
  const d = new Date(iso);
  return isNaN(d) ? String(iso ?? '—')
    : d.toLocaleString([], { month: 'short', day: 'numeric',
        hour: '2-digit', minute: '2-digit', second: '2-digit' });
 }
 function fmtMs(v) { return isNum(v) ? `${Math.round(v)} ms` : '—'; }
 function showBanner(html) { const b = $('banner'); b.innerHTML = html; b.classList.add('show'); }
 function hideBanner() { $('banner').classList.remove('show'); }
 function renderEmpty(reason) {
  $('badge').className = 'badge unknown';
  $('badgeText').textContent = 'NO DATA';
  $('statusSub').textContent = reason || 'Waiting for the first collection…';
  $('uptime').textContent = '—';
  $('uptimeSub').textContent = '— up / — checks';
  $('ssl').textContent = '—';
  $('sslCard').className = 'card';
  $('sslPill').style.display = 'none';
  $('latency').textContent = '—';
  $('latencySub').textContent = 'search — · dns —';
  $('rows').innerHTML = '';
  $('tableEmpty').style.display = 'flex';
  $('chartRange').textContent = '';
  $('chartEmpty').textContent = 'No data yet — start the collector to gather metrics.';
  $('chartEmpty').style.display = 'flex';
  $('chart').style.display = 'none';
  if (chart) { chart.destroy(); chart = null; }
 }
 function renderChart(points) {
  if (points.length === 0) {
    $('chartEmpty').textContent = 'No checks in the last hour.';
    $('chartEmpty').style.display = 'flex';
    $('chart').style.display = 'none';
    if (chart) { chart.destroy(); chart = null; }
    return;
  }
  $('chartEmpty').style.display = 'none';
  $('chart').style.display = 'block';
  const labels = points.map(p => fmtTime(p.ts));
  // Failed checks carry null, which draws a gap rather than a misleading zero.
  const latency = points.map(p => (isNum(p.latency_ms) ? p.latency_ms : null));
  const search  = points.map(p => (isNum(p.search_ms)  ? p.search_ms  : null));
  const pointColors = points.map(p => (p.up ? '#58a6ff' : '#f85149'));
  if (chart) {
    chart.data.labels = labels;
    chart.data.datasets[0].data = latency;
    chart.data.datasets[0].pointBackgroundColor = pointColors;
    chart.data.datasets[1].data = search;
    chart.update('none');
    return;
  }
  const ctx = $('chart').getContext('2d');
  const grad = ctx.createLinearGradient(0, 0, 0, 300);
  grad.addColorStop(0, 'rgba(88,166,255,.26)');
  grad.addColorStop(1, 'rgba(88,166,255,0)');
  chart = new Chart(ctx, {
    type: 'line',
    data: {
      labels,
      datasets: [
        {
          label: 'page latency (ms)', data: latency,
          borderColor: '#58a6ff', backgroundColor: grad,
          borderWidth: 2, pointRadius: 2.5, pointHoverRadius: 5,
          pointBackgroundColor: pointColors, pointBorderWidth: 0,
          tension: .3, fill: true, spanGaps: false,
        },
        {
          label: 'search latency (ms)', data: search,
          borderColor: '#bc8cff', backgroundColor: 'transparent',
          borderWidth: 2, borderDash: [5, 4],
          pointRadius: 2, pointHoverRadius: 5, pointBackgroundColor: '#bc8cff',
          pointBorderWidth: 0, tension: .3, fill: false, spanGaps: false,
        }
      ]
    },
    options: {
      responsive: true, maintainAspectRatio: false, animation: false,
      interaction: { mode: 'index', intersect: false },
      plugins: {
        legend: { display: false },
        tooltip: {
          backgroundColor: '#1c2129', borderColor: '#262d38', borderWidth: 1,
          titleColor: '#e6edf3', bodyColor: '#8b949e', padding: 10,
          callbacks: {
            label: (c) => c.parsed.y === null
              ? `${c.dataset.label.split(' ')[0]}: failed`
              : `${c.dataset.label.split(' ')[0]}: ${c.parsed.y.toFixed(0)} ms`
          }
        }
      },
      scales: {
        x: { grid: { color: 'rgba(255,255,255,.05)' },
             ticks: { color: '#8b949e', maxTicksLimit: 10, maxRotation: 0, autoSkip: true } },
        y: { beginAtZero: true, grid: { color: 'rgba(255,255,255,.05)' },
             ticks: { color: '#8b949e', callback: (v) => v + ' ms' } }
      }
    }
  });
 }
 function renderTable(records) {
  const last10 = records.slice(-10).reverse();
  const tbody = $('rows');
  if (last10.length === 0) {
    tbody.innerHTML = '';
    $('tableEmpty').style.display = 'flex';
    return;
  }
  $('tableEmpty').style.display = 'none';
  tbody.innerHTML = last10.map(r => {
    const cell = (v) => isNum(v)
      ? `<td class="num">${Math.round(v)} ms</td>`
      : `<td class="null">—</td>`;
    const code = r.code == null
      ? '<td class="null">—</td>'
      : `<td class="${r.up ? 'code-ok' : 'code-bad'}">${r.code}</td>`;
    return `<tr>
      <td class="null">${fmtTime(r.ts)}</td>
      <td>${r.up ? '✅' : '❌'}</td>
      ${code}
      ${cell(r.latency_ms)}
      ${cell(r.dns_ms)}
      ${cell(r.search_ms)}
    </tr>`;
  }).join('');
 }
 function renderSSL(records) {
  // Use the most recent check that actually managed to read the certificate.
  let days = null;
  for (let i = records.length - 1; i >= 0; i--) {
    if (isNum(records[i].ssl_days_left)) { days = records[i].ssl_days_left; break; }
  }
  const card = $('sslCard');
  const pill = $('sslPill');
  if (days === null) {
    $('ssl').textContent = '—';
    card.className = 'card';
    pill.style.display = 'none';
    return;
  }
  $('ssl').textContent = days;
  let state, text;
  if (days > 30)      { state = 'ssl-ok';   text = 'Healthy'; }
  else if (days >= 8) { state = 'ssl-warn'; text = 'Renew soon'; }
  else                { state = 'ssl-crit'; text = days <= 0 ? 'Expired' : 'Expiring'; }
  card.className = 'card ' + state;
  pill.className = 'ssl-pill';
  pill.textContent = text;
  pill.style.display = 'inline-block';
 }
 function render(raw) {
  if (!Array.isArray(raw) || raw.length === 0) { renderEmpty('No checks recorded yet.'); return; }
  const records = raw.filter(r => r && typeof r === 'object' && r.ts);
  if (records.length === 0) { renderEmpty('Data file contains no usable checks.'); return; }
  const latest = records[records.length - 1];
  const now = Date.now();
  // Status badge
  $('badge').className = 'badge ' + (latest.up ? 'up' : 'down');
  $('badgeText').textContent = latest.up ? 'UP' : 'DOWN';
  $('statusSub').textContent =
    (latest.code != null ? `HTTP ${latest.code}` : 'no response') + ' · ' + fmtDateTime(latest.ts);
  // Uptime % over the last 24h
  const window24 = records.filter(r => now - new Date(r.ts).getTime() <= UPTIME_WINDOW_MS);
  if (window24.length > 0) {
    const up = window24.filter(r => r.up === true).length;
    $('uptime').textContent = ((up / window24.length) * 100).toFixed(1) + '%';
    $('uptimeSub').textContent = `${up} up / ${window24.length} checks`;
  } else {
    $('uptime').textContent = '—';
    $('uptimeSub').textContent = 'no checks in the last 24h';
  }
  // Latest response times
  $('latency').textContent = fmtMs(latest.latency_ms);
  $('latencySub').textContent = `search ${fmtMs(latest.search_ms)} · dns ${fmtMs(latest.dns_ms)}`;
  renderSSL(records);
  const hour = records.filter(r => now - new Date(r.ts).getTime() <= CHART_WINDOW_MS);
  $('chartRange').textContent = hour.length > 1
    ? `${fmtTime(hour[0].ts)} → ${fmtTime(hour[hour.length - 1].ts)} · ${hour.length} checks`
    : (hour.length === 1 ? `${fmtTime(hour[0].ts)} · 1 check` : '');
  renderChart(hour);
  renderTable(records);
 }
 async function load() {
  try {
    const res = await fetch(`${DATA_URL}?t=${Date.now()}`, { cache: 'no-store' });
    if (!res.ok) throw new Error(`HTTP ${res.status}`);
    const data = await res.json();
    hideBanner();
    render(data);
  } catch (err) {
    // Missing file, invalid JSON, or a file:// fetch block — degrade, never crash.
    renderEmpty('Could not load metrics data.');
    showBanner(
      `Could not read <code>${DATA_URL}</code> (${err.message}). ` +
      `Run the collector to create it, and serve this folder over HTTP ` +
      `(<code>python -m http.server 8080</code> from <code>dashboard/</code>) ` +
      `— opening the file with <code>file://</code> blocks the fetch.`
    );
  }
  $('updated').textContent = new Date().toLocaleTimeString();
 }
 load();
 setInterval(load, REFRESH_MS);
 </script>
 </body>
 </html>
@@ -0,0 +1,26 @@
 #!/bin/sh
 # Entrypoint: run the collector in the background, serve the dashboard in the
 # foreground. The collector writes into the served directory, so the page can
 # fetch data/metrics.json from the same origin — no CORS, no second service.
 set -eu
 WEB_ROOT="${WEB_ROOT:-/app/dashboard}"
 PORT="${PORT:-8080}"
 mkdir -p "$WEB_ROOT/data"
 # Keep the collector alive: if it ever exits, restart it after a short pause
 # rather than leaving the dashboard serving frozen data.
 (
  while true; do
    python /app/collector/collect.py || echo "[start.sh] collector exited, restarting in 10s" >&2
    sleep 10
  done
 ) &
 COLLECTOR_PID=$!
 # Stop both processes on SIGTERM/SIGINT so the container shuts down promptly.
 trap 'kill "$COLLECTOR_PID" 2>/dev/null || true; exit 0' TERM INT
 echo "[start.sh] serving $WEB_ROOT on port $PORT"
 exec python -m http.server "$PORT" --directory "$WEB_ROOT" --bind 0.0.0.0
@@ -0,0 +1,66 @@
 #!/usr/bin/env bash
 set -euo pipefail
 # Deployment adapter for the GitHub Actions workflow.
 #
 # Ghaymah publicly documents `gy resource app launch` for deploying a
 # local Dockerfile/.ghaymah.json project. It does not currently document a
 # non-interactive API-token login command. The current binary exposes
 # email/password login flags and a generic `app update` command, but its public
 # help does not identify an external-image field. This adapter therefore
 # validates and prints the exact deployment handoff, then fails deliberately
 # instead of reporting a deployment that did not happen.
 app=""
 image=""
 usage() {
  echo "Usage: $0 --app <application-name> --image <registry/image:tag>" >&2
 }
 while [[ $# -gt 0 ]]; do
  case "$1" in
    --app)
      [[ $# -ge 2 ]] || { usage; exit 64; }
      app="$2"
      shift 2
      ;;
    --image)
      [[ $# -ge 2 ]] || { usage; exit 64; }
      image="$2"
      shift 2
      ;;
    *)
      echo "Unknown argument: $1" >&2
      usage
      exit 64
      ;;
  esac
 done
 [[ -n "$app" && -n "$image" ]] || { usage; exit 64; }
 echo "Ghaymah deployment handoff"
 echo "  Application: $app"
 echo "  Image URL:   $image"
 echo
 echo "Update the target application's container image URL to the immutable tag"
 echo "shown above in the Ghaymah dashboard, then validate its health endpoint."
 echo
 echo "Automated deployment is intentionally blocked because the public CLI docs"
 echo "do not specify API-token authentication or the external-image field for"
 echo "'gy resource app update'. Replace this adapter only with confirmed syntax."
 if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
  {
    echo "### Ghaymah deployment requires confirmed integration"
    echo
    echo "- Application: \`$app\`"
    echo "- Image: \`$image\`"
    echo
    echo "The image was built and pushed, but no deployment was claimed. Confirm"
    echo "the supported API/CLI command or update this image URL in the dashboard."
  } >> "$GITHUB_STEP_SUMMARY"
 fi
 exit 78
		`@@ -0,0 +1,2 @@`
							`fastapi==0.115.6`
							`uvicorn[standard]==0.34.0`