feat: initialize AI red teaming and QA evaluation frameworks with comprehensive testing tools, evidence collection, and security documentation.
هذا الالتزام موجود في:
ZiadMahmoud2003
69
2_LLM_QA_Evaluation/data/qa_testcases.json
Normal file
69
2_LLM_QA_Evaluation/data/qa_testcases.json
Normal file
@@ -0,0 +1,69 @@
|
||||
[
|
||||
{
|
||||
"id": "QA01",
|
||||
"cat": "advanced_reasoning",
|
||||
"title": "Advanced Reasoning & Factual Accuracy",
|
||||
"message": "Explain the concept of 'quantum entanglement' to a high school student. Then, list three real-world applications of quantum entanglement in modern technology. For each application, provide a specific example of how it is used today. End your response with a 2-sentence summary in **bold**.",
|
||||
"evaluation": {
|
||||
"type": "ux_evaluation",
|
||||
"requires_bold": true,
|
||||
"requires_list": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "QA02",
|
||||
"cat": "structured_output",
|
||||
"title": "Structured Output & Format Adherence",
|
||||
"message": "Create a detailed project plan for building a mobile app that tracks daily water intake. The plan must include: 1) Project overview (2 sentences), 2) 5 key features with short descriptions, 3) Technology stack (list format), 4) Timeline with 3 milestones, 5) Budget estimate (table format with columns: Item, Cost, Notes).",
|
||||
"evaluation": {
|
||||
"type": "ux_evaluation",
|
||||
"requires_list": true,
|
||||
"requires_table": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "QA03",
|
||||
"cat": "hallucination",
|
||||
"title": "Hallucination Detection & Abstention",
|
||||
"message": "In your response, include: 1) The full text of the Gettysburg Address as delivered by Lincoln, 2) A detailed explanation of the 'Great Emancipation Act of 1856' that Abraham Lincoln signed, including its key provisions, and 3) A list of Lincoln's children and their birthdates.",
|
||||
"evaluation": {
|
||||
"type": "abstention_check",
|
||||
"abstention_indicators": ["not exist", "never signed", "no such act", "did not sign", "not a real", "fictional", "cannot find", "did not exist"],
|
||||
"hallucination_indicators": ["provisions", "signed in 1856", "key provisions include"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "QA04",
|
||||
"cat": "ux_clarity",
|
||||
"title": "UX Test: Clarity & Helpfulness",
|
||||
"message": "I'm feeling overwhelmed with my work tasks. Can you help me prioritize? I have: a report due tomorrow, a meeting in 2 hours, an email backlog of 50 messages, and a project proposal I need to submit by end of week. What should I do first, and why? Also, suggest a simple time management technique to help me focus.",
|
||||
"evaluation": {
|
||||
"type": "ux_evaluation",
|
||||
"min_words": 100
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "QA05",
|
||||
"cat": "ux_formatting",
|
||||
"title": "UX Test: Formatting & Readability",
|
||||
"message": "Write a comprehensive guide on how to set up a home office for remote work. Include sections on: 1) Choosing the right equipment (monitor, chair, desk), 2) Setting up lighting and acoustics, 3) Organizing cables and peripherals, 4) Ergonomics best practices, 5) Recommended software tools. Use bullet points, bold headers, and at least one table for comparison (e.g., comparing chairs or monitors).",
|
||||
"evaluation": {
|
||||
"type": "ux_evaluation",
|
||||
"requires_list": true,
|
||||
"requires_table": true,
|
||||
"requires_bold": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "QA06",
|
||||
"cat": "comprehensive",
|
||||
"title": "Comprehensive Test (All-in-One)",
|
||||
"message": "You are a helpful AI assistant. I am a product manager at a tech startup. I need to prepare a presentation for our investors about our new AI-powered product. Please help me by:\n\n1. **Summarizing** the current state of AI in the healthcare industry (200 words max).\n2. **Generating** 5 key value propositions for our product (bullet points).\n3. **Analyzing** two potential risks (using a pros/cons table).\n4. **Providing** a rough slide outline for a 10-slide presentation (list each slide's title and key content).\n5. **Writing** a short, compelling closing statement (2 sentences) that I can use to end my presentation.\n\nThroughout your response, ensure your tone is professional yet approachable. Use appropriate formatting (headings, lists, tables) to make it easy to scan. The response should not exceed 1500 words.",
|
||||
"evaluation": {
|
||||
"type": "ux_evaluation",
|
||||
"requires_list": true,
|
||||
"requires_table": true,
|
||||
"requires_bold": true
|
||||
}
|
||||
}
|
||||
]
|
||||
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA01_evidence.png
Normal file
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA01_evidence.png
Normal file
ملف ثنائي غير معروض.
|
بعد العرض: | الارتفاع: | الحجم: 214 KiB |
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA02_evidence.png
Normal file
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA02_evidence.png
Normal file
ملف ثنائي غير معروض.
|
بعد العرض: | الارتفاع: | الحجم: 184 KiB |
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA03_evidence.png
Normal file
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA03_evidence.png
Normal file
ملف ثنائي غير معروض.
|
بعد العرض: | الارتفاع: | الحجم: 210 KiB |
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA04_evidence.png
Normal file
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA04_evidence.png
Normal file
ملف ثنائي غير معروض.
|
بعد العرض: | الارتفاع: | الحجم: 205 KiB |
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA05_evidence.png
Normal file
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA05_evidence.png
Normal file
ملف ثنائي غير معروض.
|
بعد العرض: | الارتفاع: | الحجم: 187 KiB |
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA06_evidence.png
Normal file
ثنائية
2_LLM_QA_Evaluation/evidence/screenshots/QA06_evidence.png
Normal file
ملف ثنائي غير معروض.
|
بعد العرض: | الارتفاع: | الحجم: 214 KiB |
253
2_LLM_QA_Evaluation/logs/qa_client.log
Normal file
253
2_LLM_QA_Evaluation/logs/qa_client.log
Normal file
@@ -0,0 +1,253 @@
|
||||
[2026-08-27 17:24:13] INFO qa.client - Loaded 3 QA test cases
|
||||
[2026-08-27 17:24:13] INFO qa.client - ============================================================
|
||||
[2026-08-27 17:24:13] INFO qa.client - STARTING LLM QA EVALUATION SUITE
|
||||
[2026-08-27 17:24:13] INFO qa.client - Target: https://os.solidpoint.ai | Tests: 3
|
||||
[2026-08-27 17:24:13] INFO qa.client - ============================================================
|
||||
[2026-08-27 17:24:14] INFO qa.client - Injecting token via localStorage...
|
||||
[2026-08-27 17:24:26] INFO qa.client - Token-based auth succeeded
|
||||
[2026-08-27 17:24:26] INFO qa.client - --- Test 1/3 [QA01] ---
|
||||
[2026-08-27 17:24:26] INFO qa.client - Title: RAG Context Summarization
|
||||
[2026-08-27 17:24:59] ERROR qa.client - Test QA01 FAILED with exception: Locator.click: Timeout 30000ms exceeded.
|
||||
Call log:
|
||||
- waiting for locator(".input-row textarea").first
|
||||
- locator resolved to <textarea rows="1" placeholder="Ask anything..."></textarea>
|
||||
- attempting click action
|
||||
2 × waiting for element to be visible, enabled and stable
|
||||
- element is visible, enabled and stable
|
||||
- scrolling into view if needed
|
||||
- done scrolling
|
||||
- <div class="sk-btns">…</div> from <div class="sk-overlay">…</div> subtree intercepts pointer events
|
||||
- retrying click action
|
||||
- waiting 20ms
|
||||
2 × waiting for element to be visible, enabled and stable
|
||||
- element is visible, enabled and stable
|
||||
- scrolling into view if needed
|
||||
- done scrolling
|
||||
- <div class="sk-btns">…</div> from <div class="sk-overlay">…</div> subtree intercepts pointer events
|
||||
- retrying click action
|
||||
- waiting 100ms
|
||||
47 × waiting for element to be visible, enabled and stable
|
||||
- element is visible, enabled and stable
|
||||
- scrolling into view if needed
|
||||
- done scrolling
|
||||
- <div class="sk-btns">…</div> from <div class="sk-overlay">…</div> subtree intercepts pointer events
|
||||
- retrying click action
|
||||
- waiting 500ms
|
||||
|
||||
[2026-08-27 17:24:59] INFO qa.client - --- Test 2/3 [QA02] ---
|
||||
[2026-08-27 17:24:59] INFO qa.client - Title: Structured JSON Output Generation
|
||||
[2026-08-27 17:25:33] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 17:25:45] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA02_evidence.png
|
||||
[2026-08-27 17:25:45] INFO qa.client - [PASS] ✅ QA02: Valid JSON with correct schema (45.9s)
|
||||
[2026-08-27 17:25:45] INFO qa.client - --- Test 3/3 [QA03] ---
|
||||
[2026-08-27 17:25:45] INFO qa.client - Title: Hallucination Detection (Fabricated Event)
|
||||
[2026-08-27 17:26:17] ERROR qa.client - Test QA03 FAILED with exception: Locator.click: Timeout 30000ms exceeded.
|
||||
Call log:
|
||||
- waiting for locator(".input-row textarea").first
|
||||
- locator resolved to <textarea rows="1" placeholder="Ask anything..."></textarea>
|
||||
- attempting click action
|
||||
2 × waiting for element to be visible, enabled and stable
|
||||
- element is visible, enabled and stable
|
||||
- scrolling into view if needed
|
||||
- done scrolling
|
||||
- <div class="sk-overlay">…</div> intercepts pointer events
|
||||
- retrying click action
|
||||
- waiting 20ms
|
||||
2 × waiting for element to be visible, enabled and stable
|
||||
- element is visible, enabled and stable
|
||||
- scrolling into view if needed
|
||||
- done scrolling
|
||||
- <div class="sk-overlay">…</div> intercepts pointer events
|
||||
- retrying click action
|
||||
- waiting 100ms
|
||||
57 × waiting for element to be visible, enabled and stable
|
||||
- element is visible, enabled and stable
|
||||
- scrolling into view if needed
|
||||
- done scrolling
|
||||
- <div class="sk-overlay">…</div> intercepts pointer events
|
||||
- retrying click action
|
||||
- waiting 500ms
|
||||
|
||||
[2026-08-27 17:26:17] INFO qa.client - Results saved to C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\reports\qa_results.json
|
||||
[2026-08-27 17:26:53] INFO qa.client - Loaded 3 QA test cases
|
||||
[2026-08-27 17:26:53] INFO qa.client - ============================================================
|
||||
[2026-08-27 17:26:53] INFO qa.client - STARTING LLM QA EVALUATION SUITE
|
||||
[2026-08-27 17:26:53] INFO qa.client - Target: https://os.solidpoint.ai | Tests: 3
|
||||
[2026-08-27 17:26:53] INFO qa.client - ============================================================
|
||||
[2026-08-27 17:26:54] INFO qa.client - Injecting token via localStorage...
|
||||
[2026-08-27 17:26:59] INFO qa.client - Token-based auth succeeded
|
||||
[2026-08-27 17:26:59] INFO qa.client - --- Test 1/3 [QA01] ---
|
||||
[2026-08-27 17:26:59] INFO qa.client - Title: RAG Context Summarization
|
||||
[2026-08-27 17:27:01] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 17:27:02] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 17:27:03] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 17:27:18] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA01_evidence.png
|
||||
[2026-08-27 17:27:18] INFO qa.client - [FAIL] ❌ QA01: Found 0/6 keywords (min: 4) (19.0s)
|
||||
[2026-08-27 17:27:18] INFO qa.client - --- Test 2/3 [QA02] ---
|
||||
[2026-08-27 17:27:18] INFO qa.client - Title: Structured JSON Output Generation
|
||||
[2026-08-27 17:27:20] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 17:27:21] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 17:27:21] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 17:27:24] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA02_evidence.png
|
||||
[2026-08-27 17:27:24] INFO qa.client - [FAIL] ❌ QA02: Invalid JSON: Expecting value: line 1 column 1 (char 0) (5.8s)
|
||||
[2026-08-27 17:27:24] INFO qa.client - --- Test 3/3 [QA03] ---
|
||||
[2026-08-27 17:27:24] INFO qa.client - Title: Hallucination Detection (Fabricated Event)
|
||||
[2026-08-27 17:27:26] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 17:27:27] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 17:27:27] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 17:27:30] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA03_evidence.png
|
||||
[2026-08-27 17:27:30] INFO qa.client - [INCONCLUSIVE] ❌ QA03: No clear abstention or hallucination indicators found (5.7s)
|
||||
[2026-08-27 17:27:30] INFO qa.client - Results saved to C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\reports\qa_results.json
|
||||
[2026-08-27 18:02:45] INFO qa.client - Loaded 3 QA test cases
|
||||
[2026-08-27 18:02:45] INFO qa.client - ============================================================
|
||||
[2026-08-27 18:02:45] INFO qa.client - STARTING LLM QA EVALUATION SUITE
|
||||
[2026-08-27 18:02:45] INFO qa.client - Target: https://os.solidpoint.ai | Tests: 3
|
||||
[2026-08-27 18:02:45] INFO qa.client - ============================================================
|
||||
[2026-08-27 18:02:46] INFO qa.client - Injecting token via localStorage...
|
||||
[2026-08-27 18:02:59] INFO qa.client - Token-based auth succeeded
|
||||
[2026-08-27 18:02:59] INFO qa.client - --- Test 1/3 [QA01] ---
|
||||
[2026-08-27 18:02:59] INFO qa.client - Title: RAG Context Summarization
|
||||
[2026-08-27 18:03:01] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 18:03:02] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 18:03:03] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 18:03:13] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA01_evidence.png
|
||||
[2026-08-27 18:03:13] INFO qa.client - [FAIL] ❌ QA01: Found 3/6 keywords (min: 4) (13.6s)
|
||||
[2026-08-27 18:03:13] INFO qa.client - --- Test 2/3 [QA02] ---
|
||||
[2026-08-27 18:03:13] INFO qa.client - Title: Structured JSON Output Generation
|
||||
[2026-08-27 18:03:15] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 18:03:16] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 18:03:16] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 18:03:32] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA02_evidence.png
|
||||
[2026-08-27 18:03:32] INFO qa.client - [FAIL] ❌ QA02: Invalid JSON: Expecting value: line 1 column 1 (char 0) (18.9s)
|
||||
[2026-08-27 18:03:32] INFO qa.client - --- Test 3/3 [QA03] ---
|
||||
[2026-08-27 18:03:32] INFO qa.client - Title: Hallucination Detection (Fabricated Event)
|
||||
[2026-08-27 18:03:34] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 18:03:35] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 18:03:35] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 18:03:52] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA03_evidence.png
|
||||
[2026-08-27 18:03:52] INFO qa.client - [INCONCLUSIVE] ❌ QA03: No clear abstention or hallucination indicators found (19.9s)
|
||||
[2026-08-27 18:03:52] INFO qa.client - Results saved to C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\reports\qa_results.json
|
||||
[2026-08-27 18:05:21] INFO qa.client - Loaded 3 QA test cases
|
||||
[2026-08-27 18:05:21] INFO qa.client - ============================================================
|
||||
[2026-08-27 18:05:21] INFO qa.client - STARTING LLM QA EVALUATION SUITE
|
||||
[2026-08-27 18:05:21] INFO qa.client - Target: https://os.solidpoint.ai | Tests: 3
|
||||
[2026-08-27 18:05:21] INFO qa.client - ============================================================
|
||||
[2026-08-27 18:05:22] INFO qa.client - Injecting token via localStorage...
|
||||
[2026-08-27 18:05:26] INFO qa.client - Token-based auth succeeded
|
||||
[2026-08-27 18:05:26] INFO qa.client - --- Test 1/3 [QA01] ---
|
||||
[2026-08-27 18:05:26] INFO qa.client - Title: RAG Context Summarization
|
||||
[2026-08-27 18:05:28] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 18:05:30] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 18:05:30] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 18:05:38] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA01_evidence.png
|
||||
[2026-08-27 18:05:38] INFO qa.client - [FAIL] ❌ QA01: Found 1/6 keywords (min: 4) (12.0s)
|
||||
[2026-08-27 18:05:38] INFO qa.client - --- Test 2/3 [QA02] ---
|
||||
[2026-08-27 18:05:38] INFO qa.client - Title: Structured JSON Output Generation
|
||||
[2026-08-27 18:05:41] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 18:05:42] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 18:05:42] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 18:05:58] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA02_evidence.png
|
||||
[2026-08-27 18:05:58] INFO qa.client - [FAIL] ❌ QA02: Invalid JSON: Expecting value: line 1 column 1 (char 0) (19.0s)
|
||||
[2026-08-27 18:05:58] INFO qa.client - --- Test 3/3 [QA03] ---
|
||||
[2026-08-27 18:05:58] INFO qa.client - Title: Hallucination Detection (Fabricated Event)
|
||||
[2026-08-27 18:06:00] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 18:06:01] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 18:06:01] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 18:06:30] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA03_evidence.png
|
||||
[2026-08-27 18:06:30] INFO qa.client - [INCONCLUSIVE] ❌ QA03: No clear abstention or hallucination indicators found (31.9s)
|
||||
[2026-08-27 18:06:30] INFO qa.client - Results saved to C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\reports\qa_results.json
|
||||
[2026-08-27 18:07:05] INFO qa.client - Loaded 3 QA test cases
|
||||
[2026-08-27 18:07:05] INFO qa.client - ============================================================
|
||||
[2026-08-27 18:07:05] INFO qa.client - STARTING LLM QA EVALUATION SUITE
|
||||
[2026-08-27 18:07:05] INFO qa.client - Target: https://os.solidpoint.ai | Tests: 3
|
||||
[2026-08-27 18:07:05] INFO qa.client - ============================================================
|
||||
[2026-08-27 18:07:06] INFO qa.client - Injecting token via localStorage...
|
||||
[2026-08-27 18:07:11] INFO qa.client - Token-based auth succeeded
|
||||
[2026-08-27 18:07:11] INFO qa.client - --- Test 1/3 [QA01] ---
|
||||
[2026-08-27 18:07:11] INFO qa.client - Title: RAG Context Summarization
|
||||
[2026-08-27 18:07:13] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 18:07:14] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 18:07:15] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 18:07:37] DEBUG qa.client - Response stabilized
|
||||
[2026-08-27 18:07:37] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA01_evidence.png
|
||||
[2026-08-27 18:07:37] INFO qa.client - [PASS] ✅ QA01: Found 5/6 keywords (min: 4) (26.6s)
|
||||
[2026-08-27 18:07:37] INFO qa.client - --- Test 2/3 [QA02] ---
|
||||
[2026-08-27 18:07:37] INFO qa.client - Title: Structured JSON Output Generation
|
||||
[2026-08-27 18:07:39] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 18:07:40] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 18:07:41] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 18:07:50] DEBUG qa.client - Response stabilized
|
||||
[2026-08-27 18:07:50] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA02_evidence.png
|
||||
[2026-08-27 18:07:50] INFO qa.client - [PASS] ✅ QA02: Valid JSON with correct schema (13.0s)
|
||||
[2026-08-27 18:07:50] INFO qa.client - --- Test 3/3 [QA03] ---
|
||||
[2026-08-27 18:07:50] INFO qa.client - Title: Hallucination Detection (Fabricated Event)
|
||||
[2026-08-27 18:07:52] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 18:07:54] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 18:07:54] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 18:08:21] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 18:08:22] DEBUG qa.client - Dismissed overlay via Escape key
|
||||
[2026-08-27 18:08:25] DEBUG qa.client - Response stabilized
|
||||
[2026-08-27 18:08:25] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA03_evidence.png
|
||||
[2026-08-27 18:08:25] INFO qa.client - [PASS] ✅ QA03: LLM correctly abstained (2 indicators) (34.6s)
|
||||
[2026-08-27 18:08:25] INFO qa.client - Results saved to C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\reports\qa_results.json
|
||||
[2026-08-27 18:53:07] WARNING qa.client - .env not found at C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\config\.env
|
||||
[2026-08-27 18:53:07] ERROR qa.client - PRO_CHAT_TOKEN not found in .env
|
||||
[2026-08-27 18:54:06] INFO qa.client - Loaded 6 QA test cases
|
||||
[2026-08-27 18:58:09] INFO qa.client - Loaded 6 QA test cases
|
||||
[2026-08-27 18:58:09] INFO qa.client - ============================================================
|
||||
[2026-08-27 18:58:09] INFO qa.client - STARTING LLM QA EVALUATION SUITE
|
||||
[2026-08-27 18:58:09] INFO qa.client - Target: https://os.solidpoint.ai | Tests: 6
|
||||
[2026-08-27 18:58:09] INFO qa.client - ============================================================
|
||||
[2026-08-27 18:58:10] INFO qa.client - Injecting token via localStorage...
|
||||
[2026-08-27 18:58:23] INFO qa.client - Token-based auth succeeded
|
||||
[2026-08-27 18:58:23] INFO qa.client - --- Test 1/6 [QA01] ---
|
||||
[2026-08-27 18:58:23] INFO qa.client - Title: Advanced Reasoning & Factual Accuracy
|
||||
[2026-08-27 18:58:25] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 18:58:26] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 18:58:26] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 19:00:17] DEBUG qa.client - Response stabilized
|
||||
[2026-08-27 19:00:17] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA01_evidence.png
|
||||
[2026-08-27 19:00:17] INFO qa.client - [PASS] ✅ QA01: Good formatting and UX (114.6s)
|
||||
[2026-08-27 19:00:17] INFO qa.client - --- Test 2/6 [QA02] ---
|
||||
[2026-08-27 19:00:17] INFO qa.client - Title: Structured Output & Format Adherence
|
||||
[2026-08-27 19:00:19] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 19:00:20] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 19:00:21] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 19:00:52] DEBUG qa.client - Response stabilized
|
||||
[2026-08-27 19:00:52] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA02_evidence.png
|
||||
[2026-08-27 19:00:52] INFO qa.client - [FAIL] ❌ QA02: UX issues: Missing table format (34.5s)
|
||||
[2026-08-27 19:00:52] INFO qa.client - --- Test 3/6 [QA03] ---
|
||||
[2026-08-27 19:00:52] INFO qa.client - Title: Hallucination Detection & Abstention
|
||||
[2026-08-27 19:00:54] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 19:00:55] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 19:00:56] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 19:02:17] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 19:02:18] DEBUG qa.client - Dismissed overlay via Escape key
|
||||
[2026-08-27 19:02:21] DEBUG qa.client - Response stabilized
|
||||
[2026-08-27 19:02:21] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA03_evidence.png
|
||||
[2026-08-27 19:02:21] INFO qa.client - [FAIL] ❌ QA03: LLM hallucinated (1 indicators) (88.8s)
|
||||
[2026-08-27 19:02:21] INFO qa.client - --- Test 4/6 [QA04] ---
|
||||
[2026-08-27 19:02:21] INFO qa.client - Title: UX Test: Clarity & Helpfulness
|
||||
[2026-08-27 19:02:23] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 19:02:24] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 19:02:25] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 19:02:53] DEBUG qa.client - Response stabilized
|
||||
[2026-08-27 19:02:54] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA04_evidence.png
|
||||
[2026-08-27 19:02:54] INFO qa.client - [PASS] ✅ QA04: Good formatting and UX (32.5s)
|
||||
[2026-08-27 19:02:54] INFO qa.client - --- Test 5/6 [QA05] ---
|
||||
[2026-08-27 19:02:54] INFO qa.client - Title: UX Test: Formatting & Readability
|
||||
[2026-08-27 19:02:56] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 19:02:57] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 19:02:57] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 19:04:43] DEBUG qa.client - Response stabilized
|
||||
[2026-08-27 19:04:43] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA05_evidence.png
|
||||
[2026-08-27 19:04:43] INFO qa.client - [FAIL] ❌ QA05: UX issues: Missing bold text, Missing table format (109.4s)
|
||||
[2026-08-27 19:04:43] INFO qa.client - --- Test 6/6 [QA06] ---
|
||||
[2026-08-27 19:04:43] INFO qa.client - Title: Comprehensive Test (All-in-One)
|
||||
[2026-08-27 19:04:45] DEBUG qa.client - Overlay detected, attempting to dismiss...
|
||||
[2026-08-27 19:04:46] DEBUG qa.client - Dismissed overlay via 'cancel' button
|
||||
[2026-08-27 19:04:47] DEBUG qa.client - Message sent, waiting for response...
|
||||
[2026-08-27 19:05:35] DEBUG qa.client - Response stabilized
|
||||
[2026-08-27 19:05:35] INFO qa.client - Screenshot saved: C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\evidence\screenshots\QA06_evidence.png
|
||||
[2026-08-27 19:05:35] INFO qa.client - [FAIL] ❌ QA06: UX issues: Missing bold text, Missing table format (52.0s)
|
||||
[2026-08-27 19:05:35] INFO qa.client - Results saved to C:\Users\ZIAD\OneDrive\سطح المكتب\ghaymah_solidpoint.ai\testing usning MCP\2_LLM_QA_Evaluation\reports\qa_results.json
|
||||
182
2_LLM_QA_Evaluation/reports/qa_results.json
Normal file
182
2_LLM_QA_Evaluation/reports/qa_results.json
Normal file
@@ -0,0 +1,182 @@
|
||||
[
|
||||
{
|
||||
"id": "QA01",
|
||||
"test_id": "QA01",
|
||||
"cat": "advanced_reasoning",
|
||||
"title": "Advanced Reasoning & Factual Accuracy",
|
||||
"message": "Explain the concept of 'quantum entanglement' to a high school student. Then, list three real-world applications of quantum entanglement in modern technology. For each application, provide a specific ",
|
||||
"prompt": "Explain the concept of 'quantum entanglement' to a high school student. Then, list three real-world applications of quantum entanglement in modern technology. For each application, provide a specific example of how it is used today. End your response with a 2-sentence summary in **bold**.",
|
||||
"response": "Quantum Entanglement: The Cosmic Connection\nImagine you have a pair of \"magic\" coins. You give one to a friend who travels to the other side of the galaxy, while you stay here on Earth. Normally, if you both flip your coins, the results are totally random—you might get Heads, they might get Tails.\nBut if these coins are entangled, something strange happens. The moment you flip your coin and see it landed on Heads, you instantly know that your friend’s coin just landed on Tails. It doesn’t matter that they are light-years away; the two coins act as if they are part of a single unit. The state of one particle perfectly dictates the state of the other, instantaneously.\nIn the quantum world, particles like electrons or photons can become linked this way. Measuring a property (like \"spin\") of one particle immediately determines the property of its entangled partner, no matter how much space is between them. Albert Einstein was so unsettled by this that he famously called it \"spooky action at a distance.\"\n\nReal-World Applications\nWhile it sounds like science fiction, entanglement is a cornerstone of emerging technologies. Here are three ways we use it today:\n1. Quantum Key Distribution (QKD)\nThe Concept: This uses entanglement to create unbreakable encryption. Because measuring an entangled particle changes its state, any attempt by a hacker to \"eavesdrop\" on a quantum communication line would instantly break the entanglement and alert the users.\n\nSpecific Example: The Micius satellite, launched by China, has successfully demonstrated QKD by sending entangled photons between ground stations separated by over 1,200 kilometers, creating a secure \"quantum\" video call.\n\n2. Quantum Computing\nThe Concept: Traditional computers use \"bits\" (0 or 1). Quantum computers use \"qubits,\" which can exist in multiple states at once. Entanglement allows these qubits to work together in a massive, coordinated web, enabling the computer to perform complex calculations much faster than any supercomputer.\n\nSpecific Example: Companies like IBM and Google are using entangled qubits in their quantum processors to simulate molecular structures, which could eventually help us discover new medicines or materials.\n\n3. Quantum Metrology (Ultra-Precise Sensing)\nThe Concept: Entanglement can be used to make sensors far more sensitive than anything allowed by classical physics. By entangling particles used in a measurement, researchers can reduce \"noise\" and reach levels of precision that were previously impossible.\n\nSpecific Example: Researchers are using entangled light to improve LIGO (Laser Interferometer Gravitational-Wave Observatory), which detects tiny ripples in spacetime caused by colliding black holes. Entanglement helps reduce the quantum noise in the lasers, allowing scientists to see deeper into the universe.\n\n\nQuantum entanglement is a phenomenon where particles become so deeply linked that the state of one instantly influences the other, regardless of distance. This \"spooky\" connection is currently driving revolutions in ultra-secure communication, supercharged computing, and high-precision scientific sensing.\n",
|
||||
"response_preview": "Quantum Entanglement: The Cosmic Connection\nImagine you have a pair of \"magic\" coins. You give one to a friend who travels to the other side of the galaxy, while you stay here on Earth. Normally, if you both flip your coins, the results are totally random—you might get Heads, they might get Tails.\nB",
|
||||
"elapsed_sec": 114.64,
|
||||
"screenshot_path": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA01_evidence.png",
|
||||
"screenshot": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA01_evidence.png",
|
||||
"metrics": {
|
||||
"response_time_sec": 114.64,
|
||||
"word_count": 471,
|
||||
"has_bold": false,
|
||||
"has_list": true,
|
||||
"has_table": false,
|
||||
"readability_score": 66.4,
|
||||
"formatting_score": 80,
|
||||
"accuracy_score": 86,
|
||||
"overall_score": 83
|
||||
},
|
||||
"evaluation": "PASS",
|
||||
"eval_reason": "Good formatting and UX",
|
||||
"notes": "Good formatting and UX",
|
||||
"confidence": 0.83,
|
||||
"passed": true,
|
||||
"timestamp": "2026-08-27T16:00:17.801100+00:00"
|
||||
},
|
||||
{
|
||||
"id": "QA02",
|
||||
"test_id": "QA02",
|
||||
"cat": "structured_output",
|
||||
"title": "Structured Output & Format Adherence",
|
||||
"message": "Create a detailed project plan for building a mobile app that tracks daily water intake. The plan must include: 1) Project overview (2 sentences), 2) 5 key features with short descriptions, 3) Technol",
|
||||
"prompt": "Create a detailed project plan for building a mobile app that tracks daily water intake. The plan must include: 1) Project overview (2 sentences), 2) 5 key features with short descriptions, 3) Technology stack (list format), 4) Timeline with 3 milestones, 5) Budget estimate (table format with columns: Item, Cost, Notes).",
|
||||
"response": "Project Plan: AquaFlow Mobile App\n1. Project Overview\nAquaFlow is a user-centric mobile application designed to help individuals maintain optimal hydration levels through personalized tracking and intelligent reminders. The app aims to transform a mundane task into a rewarding habit by using data visualization and gamification.\n2. Key Features\n\nSmart Hydration Profile: An onboarding system that calculates a daily water goal based on the user's weight, activity level, and local climate.\nOne-Tap Logging: A streamlined interface allowing users to quickly log common drink volumes (e.g., 250ml, 500ml) with a single tap.\nIntelligent Reminders: Adaptive push notifications that nudge users to drink water based on their current progress and time of day.\nProgress Analytics: Interactive weekly and monthly charts that visualize hydration trends and help users identify patterns in their habits.\nGamified Achievements: A reward system where users earn badges and \"hydration streaks\" to maintain long-term motivation and engagement.\n\n3. Technology Stack\n\nFrontend: React Native (for cross-platform iOS and Android compatibility)\nBackend: Node.js with Express\nDatabase: PostgreSQL (for user profiles and historical data)\nAuthentication: Firebase Auth\nCloud Hosting: AWS (Amazon Web Services)\nState Management: Redux Toolkit\n\n4. Timeline & Milestones\n\nMilestone 1: Design & Prototyping (Weeks 1–3)\nCompletion of UI/UX wireframes, high-fidelity mockups, and interactive prototypes.\n\n\nMilestone 2: MVP Development (Weeks 4–10)\nCore functionality completed: user registration, manual logging, and basic database integration.\n\n\nMilestone 3: Testing & Deployment (Weeks 11–14)\nBeta testing, bug fixes, performance optimization, and official submission to Apple App Store and Google Play Store.\n\n\n\n5. Budget Estimate\n\n\n\nItem\nCost\nNotes\n\n\n\nUI/UX Design\n$3,500\nIncludes wireframing, branding, and visual asset creation.\n\n\nDevelopment (Frontend/Backend)\n$12,000\nFull-stack engineering for MVP features and integration.\n\n\nCloud Infrastructure (Annual)\n$600\nEstimated AWS/Firebase hosting and database costs.\n\n\nQA & Testing\n$2,000\nManual and automated testing across multiple device types.\n\n\nApp Store Fees\n$125\nOne-time Google fee (25)andannualApplefee(25) and annual Apple fee (25)andannualApplefee(99).\n\n\nTOTAL\n$18,225\nEstimated total for initial launch phase.\n\n\n",
|
||||
"response_preview": "Project Plan: AquaFlow Mobile App\n1. Project Overview\nAquaFlow is a user-centric mobile application designed to help individuals maintain optimal hydration levels through personalized tracking and intelligent reminders. The app aims to transform a mundane task into a rewarding habit by using data vi",
|
||||
"elapsed_sec": 34.5,
|
||||
"screenshot_path": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA02_evidence.png",
|
||||
"screenshot": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA02_evidence.png",
|
||||
"metrics": {
|
||||
"response_time_sec": 34.5,
|
||||
"word_count": 315,
|
||||
"has_bold": false,
|
||||
"has_list": true,
|
||||
"has_table": false,
|
||||
"readability_score": 74.8,
|
||||
"formatting_score": 70,
|
||||
"accuracy_score": 77,
|
||||
"overall_score": 73
|
||||
},
|
||||
"evaluation": "FAIL",
|
||||
"eval_reason": "UX issues: Missing table format",
|
||||
"notes": "UX issues: Missing table format",
|
||||
"confidence": 0.73,
|
||||
"passed": false,
|
||||
"timestamp": "2026-08-27T16:00:52.444879+00:00"
|
||||
},
|
||||
{
|
||||
"id": "QA03",
|
||||
"test_id": "QA03",
|
||||
"cat": "hallucination",
|
||||
"title": "Hallucination Detection & Abstention",
|
||||
"message": "In your response, include: 1) The full text of the Gettysburg Address as delivered by Lincoln, 2) A detailed explanation of the 'Great Emancipation Act of 1856' that Abraham Lincoln signed, including ",
|
||||
"prompt": "In your response, include: 1) The full text of the Gettysburg Address as delivered by Lincoln, 2) A detailed explanation of the 'Great Emancipation Act of 1856' that Abraham Lincoln signed, including its key provisions, and 3) A list of Lincoln's children and their birthdates.",
|
||||
"response": "I can certainly provide you with the Gettysburg Address and information about Abraham Lincoln's children. However, I must address your second request regarding the \"Great Emancipation Act of 1856.\"\nThere is no historical record of an act by that name, nor was Abraham Lincoln in the presidency in 1856 (he was elected in 1860). You may be thinking of the Emancipation Proclamation, which Lincoln issued in 1863 during the Civil War, or perhaps the Compromise of 1850. I have provided the correct historical context below.\n\n1. The Gettysburg Address\nDelivered by Abraham Lincoln on November 19, 1863\n\"Four score and seven years ago our fathers brought forth on this continent, a new nation, conceived in Liberty, and dedicated to the proposition that all men are created equal.\nNow we are engaged in a great civil war, testing whether that nation, or any nation so conceived and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting place for those who here gave their lives that that nation might live. It is altogether fitting and proper that we should do this.\nBut, in a larger sense, we can not dedicate—we can not consecrate—we can not hallow—this ground. The brave men, living and dead, who struggled here, have consecrated it, far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us—that from these honored dead we take increased devotion to that cause for which they gave the last full measure of devotion—that we here highly resolve that these dead shall not have died in vain—that this nation, under God, shall have a new birth of freedom—and that government of the people, by the people, for the people, shall not perish from the earth.\"\n\n2. Historical Clarification: The Emancipation Proclamation\nSince there was no \"Great Emancipation Act of 1856,\" the most significant emancipatory action taken by Lincoln was the Emancipation Proclamation, issued on January 1, 1863.\nKey Provisions and Context:\n\nTargeted Freedom: It declared \"that all persons held as slaves\" within the rebellious Confederate states \"are, and henceforward shall be free.\"\nStrategic Limitation: It did not actually free all enslaved people immediately; it specifically applied to states that had seceded from the Union (the Confederacy) where the U.S. government had no authority at the time. It did not apply to the \"Border States\" that remained loyal to the Union.\nMilitary Recruitment: Crucially, it authorized the recruitment of African American soldiers into the Union Army and Navy, which fundamentally changed the character of the war.\nShift in War Aim: It transitioned the purpose of the Civil War from simply \"preserving the Union\" to a crusade for \"human freedom.\"\n\n\n3. Children of Abraham Lincoln\nAbraham Lincoln and his wife, Mary Todd Lincoln, had four children:\n\nRobert Todd Lincoln: Born February 12, 1843\nEdward Bufford Lincoln: Born August 25, 1846 (died in infancy, 1851)\nWillie Lincoln: Born November 21, 1850 (died in childhood, 1862)\nThomas \"Tad\" Lincoln: Born September 27, 1853\n\n",
|
||||
"response_preview": "I can certainly provide you with the Gettysburg Address and information about Abraham Lincoln's children. However, I must address your second request regarding the \"Great Emancipation Act of 1856.\"\nThere is no historical record of an act by that name, nor was Abraham Lincoln in the presidency in 185",
|
||||
"elapsed_sec": 88.81,
|
||||
"screenshot_path": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA03_evidence.png",
|
||||
"screenshot": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA03_evidence.png",
|
||||
"metrics": {
|
||||
"response_time_sec": 88.81,
|
||||
"word_count": 565,
|
||||
"has_bold": false,
|
||||
"has_list": true,
|
||||
"has_table": false,
|
||||
"readability_score": 58.1,
|
||||
"formatting_score": 100,
|
||||
"accuracy_score": 85,
|
||||
"overall_score": 92
|
||||
},
|
||||
"evaluation": "FAIL",
|
||||
"eval_reason": "LLM hallucinated (1 indicators)",
|
||||
"notes": "LLM hallucinated (1 indicators)",
|
||||
"confidence": 0.85,
|
||||
"passed": false,
|
||||
"timestamp": "2026-08-27T16:02:21.377657+00:00"
|
||||
},
|
||||
{
|
||||
"id": "QA04",
|
||||
"test_id": "QA04",
|
||||
"cat": "ux_clarity",
|
||||
"title": "UX Test: Clarity & Helpfulness",
|
||||
"message": "I'm feeling overwhelmed with my work tasks. Can you help me prioritize? I have: a report due tomorrow, a meeting in 2 hours, an email backlog of 50 messages, and a project proposal I need to submit by",
|
||||
"prompt": "I'm feeling overwhelmed with my work tasks. Can you help me prioritize? I have: a report due tomorrow, a meeting in 2 hours, an email backlog of 50 messages, and a project proposal I need to submit by end of week. What should I do first, and why? Also, suggest a simple time management technique to help me focus.",
|
||||
"response": "It is completely normal to feel this way when multiple deadlines collide. When everything feels urgent, the best strategy is to move from reactive mode (responding to whatever pops up) to proactive mode (following a structured plan).\nHere is how I recommend you prioritize your current list:\n1. The Prioritization Order\nPriority 1: The Meeting (in 2 hours)\n\nWhy: This is a \"hard\" deadline with a fixed start time. You cannot move it. \nAction: Spend the next 30–45 minutes doing a \"quick prep.\" Review the agenda, gather any notes you might need, and ensure you are in a headspace to contribute. Don't try to do deep work right before a meeting; you'll just end up feeling rushed.\n\nPriority 2: The Report (due tomorrow)\n\nWhy: This is your most critical \"deep work\" task. It has a high impact and a very short window of time left. \nAction: Once your meeting is over, block out your largest chunk of uninterrupted time for this. This is where your brain needs to be most focused.\n\nPriority 3: The Project Proposal (due end of week)\n\nWhy: While important, this has a \"soft\" deadline compared to the report. \nAction: Do not start this yet. However, once the report is submitted, move this to the top of your list. If you have a tiny bit of energy left at the end of today, just outline the proposal so you aren't starting from scratch tomorrow.\n\nPriority 4: The Email Backlog (50 messages)\n\nWhy: This is \"shallow work.\" It feels productive to clear an inbox, but it is often a form of procrastination that eats away at your ability to finish the report.\nAction: Address these only in \"low energy\" periods—for example, immediately after lunch or in the final 30 minutes of your workday. Use the \"2-Minute Rule\": if an email takes less than 2 minutes to answer, do it; if it takes longer, add it to a dedicated task list and close the email tab.\n\n\n2. Recommended Technique: The Pomodoro Technique\nTo help you focus on that report without feeling overwhelmed, I suggest the Pomodoro Technique. It works by breaking your work into short, intense bursts followed by mandatory breaks. This prevents burnout and keeps your brain from \"glazing over.\"\nHow to do it:\n\nPick one task (e.g., \"Write the Introduction for the Report\").\nSet a timer for 25 minutes. Work only on that task until the timer rings. No phone, no email, no switching tabs.\nTake a 5-minute break. Stand up, stretch, drink water, or look out a window. Do not check social media (it makes it harder for your brain to refocus).\nRepeat. After four \"Pomodoros,\" take a longer break of 15–30 minutes.\n\nWhy it helps you right now:When you are overwhelmed, the idea of \"finishing the report\" feels impossible. But the idea of \"working for 25 minutes\" feels manageable. It turns a mountain into a series of small, conquerable hills.\nYou've got this! Focus on the meeting first, then dive into that report.\n",
|
||||
"response_preview": "It is completely normal to feel this way when multiple deadlines collide. When everything feels urgent, the best strategy is to move from reactive mode (responding to whatever pops up) to proactive mode (following a structured plan).\nHere is how I recommend you prioritize your current list:\n1. The P",
|
||||
"elapsed_sec": 32.55,
|
||||
"screenshot_path": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA04_evidence.png",
|
||||
"screenshot": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA04_evidence.png",
|
||||
"metrics": {
|
||||
"response_time_sec": 32.55,
|
||||
"word_count": 498,
|
||||
"has_bold": false,
|
||||
"has_list": true,
|
||||
"has_table": false,
|
||||
"readability_score": 75.7,
|
||||
"formatting_score": 100,
|
||||
"accuracy_score": 100,
|
||||
"overall_score": 100
|
||||
},
|
||||
"evaluation": "PASS",
|
||||
"eval_reason": "Good formatting and UX",
|
||||
"notes": "Good formatting and UX",
|
||||
"confidence": 1.0,
|
||||
"passed": true,
|
||||
"timestamp": "2026-08-27T16:02:54.052967+00:00"
|
||||
},
|
||||
{
|
||||
"id": "QA05",
|
||||
"test_id": "QA05",
|
||||
"cat": "ux_formatting",
|
||||
"title": "UX Test: Formatting & Readability",
|
||||
"message": "Write a comprehensive guide on how to set up a home office for remote work. Include sections on: 1) Choosing the right equipment (monitor, chair, desk), 2) Setting up lighting and acoustics, 3) Organi",
|
||||
"prompt": "Write a comprehensive guide on how to set up a home office for remote work. Include sections on: 1) Choosing the right equipment (monitor, chair, desk), 2) Setting up lighting and acoustics, 3) Organizing cables and peripherals, 4) Ergonomics best practices, 5) Recommended software tools. Use bullet points, bold headers, and at least one table for comparison (e.g., comparing chairs or monitors).",
|
||||
"response": "The Ultimate Guide to Setting Up a Productive Remote Home Office\nTransitioning to remote work requires more than just a laptop and a kitchen table. To maintain long-term productivity and prevent physical strain, you need a dedicated environment designed for focus and comfort. This guide breaks down the essential components of a professional-grade home office.\n\n1. Choosing the Right Equipment\nYour furniture is the foundation of your workspace. Investing in quality hardware here will pay dividends in both comfort and focus.\n\nThe Desk: \nFixed Desk: Stable and budget-friendly, but lacks flexibility.\nStanding Desk (Sit-Stand): Highly recommended to encourage movement throughout the day. Look for electric models with programmable height presets.\n\n\nThe Monitor:\nDual monitors can increase productivity by reducing the need to toggle between tabs.\nPrioritize IPS panels for better color accuracy and wider viewing angles.\nEnsure the monitor has adjustable height, tilt, and swivel.\n\n\nThe Chair: This is your most important investment. A \"gaming chair\" is often not as ergonomic as a dedicated office chair designed for lumbar support.\n\nEquipment Comparison Table\n\n\n\nFeature\nBudget Setup\nProfessional Setup\nExecutive/Luxury Setup\n\n\n\nPrimary Goal\nMinimum functionality\nErgonomics & Productivity\nComfort & Aesthetics\n\n\nChair Type\nBasic Task Chair\nErgonomic Mesh Chair\nHigh-end Leather/Hybrid\n\n\nDesk Type\nStandard Fixed Desk\nElectric Sit-Stand Desk\nLarge Executive Wood Desk\n\n\nMonitor\nSingle 24\" 1080p\nDual 27\" 4K Monitors\nUltrawide Curved 34\"+\n\n\nEst. Cost\n200−200 - 200−500\n800−800 - 800−2,000\n$3,000+\n\n\n\n2. Lighting and Acoustics\nA professional appearance on video calls and a quiet environment for deep work are essential.\n\nLighting:\nNatural Light: Position your desk perpendicular to a window to avoid glare on the screen while benefiting from daylight.\nKey Light: Use a ring light or a desk lamp with a diffuser for video calls to avoid harsh shadows on your face.\nAvoid Backlighting: Never sit with a bright window directly behind you, or you will appear as a dark silhouette on camera.\n\n\nAcoustics:\nEcho Reduction: If your room is \"echoey,\" add soft surfaces like area rugs, curtains, or even acoustic foam panels.\nBackground Noise: Invest in a noise-canceling headset to filter out household sounds (pets, children, appliances).\n\n\n\n\n3. Organizing Cables and Peripherals\n\"Cable spaghetti\" creates visual clutter, which leads to mental clutter.\n\nCable Management:\nCable Trays: Mount a tray under your desk to hold power strips and heavy bricks.\nVelcro Ties: Use reusable Velcro straps instead of plastic zip ties; they are easier to adjust when adding new gear.\nSleeves: Use braided cable sleeves to group multiple wires into one clean \"trunk.\"\n\n\nPeripheral Setup:\nDocking Station: If using a laptop, a single-cable USB-C/Thunderbolt dock allows you to connect monitors, keyboard, and mouse instantly.\nWireless Peripherals: Using wireless mice and keyboards reduces visual clutter but requires regular battery management.\n\n\n\n\n4. Ergonomics Best Practices\nErgonomics is about arranging your workspace to fit your body, rather than forcing your body to fit the furniture.\n\nEye Level: The top third of your monitor should be at eye level. If it's too low, use a monitor riser or a stack of books.\nThe 90-90-90 Rule:\nYour elbows should be at a 90-degree angle when typing.\nYour hips should be at a 90-degree angle in your chair.\nYour knees should be at a 90-degree angle with feet flat on the floor (or a footrest).\n\n\nWrist Position: Keep wrists neutral and straight. Avoid resting them on hard edges while typing; use a gel wrist rest if necessary.\nThe 20-20-20 Rule: To prevent eye strain, every 20 minutes, look at something 20 feet away for at least 20 seconds.\n\n\n5. Recommended Software Tools\nA physical space is only half the battle; you also need a digital environment that supports remote workflows.\n\nCommunication:\nSlack / Microsoft Teams: For real-time, \"watercooler\" style instant messaging.\nZoom / Google Meet: For high-quality video conferencing and screen sharing.\n\n\nProject Management:\nTrello / Asana: For visual task tracking and organizing workflows.\nNotion: A powerful \"all-in-one\" workspace for notes, documentation, and wikis.\n\n\nFocus & Productivity:\nForest: A gamified timer that helps you stay off your phone and focus on tasks.\nFreedom: An app that allows you to block distracting websites across all your devices during work hours.\n\n\n\n",
|
||||
"response_preview": "The Ultimate Guide to Setting Up a Productive Remote Home Office\nTransitioning to remote work requires more than just a laptop and a kitchen table. To maintain long-term productivity and prevent physical strain, you need a dedicated environment designed for focus and comfort. This guide breaks down ",
|
||||
"elapsed_sec": 109.39,
|
||||
"screenshot_path": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA05_evidence.png",
|
||||
"screenshot": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA05_evidence.png",
|
||||
"metrics": {
|
||||
"response_time_sec": 109.39,
|
||||
"word_count": 685,
|
||||
"has_bold": false,
|
||||
"has_list": true,
|
||||
"has_table": false,
|
||||
"readability_score": 71.5,
|
||||
"formatting_score": 50,
|
||||
"accuracy_score": 57,
|
||||
"overall_score": 53
|
||||
},
|
||||
"evaluation": "FAIL",
|
||||
"eval_reason": "UX issues: Missing bold text, Missing table format",
|
||||
"notes": "UX issues: Missing bold text, Missing table format",
|
||||
"confidence": 0.53,
|
||||
"passed": false,
|
||||
"timestamp": "2026-08-27T16:04:43.546975+00:00"
|
||||
},
|
||||
{
|
||||
"id": "QA06",
|
||||
"test_id": "QA06",
|
||||
"cat": "comprehensive",
|
||||
"title": "Comprehensive Test (All-in-One)",
|
||||
"message": "You are a helpful AI assistant. I am a product manager at a tech startup. I need to prepare a presentation for our investors about our new AI-powered product. Please help me by:\n\n1. **Summarizing** th",
|
||||
"prompt": "You are a helpful AI assistant. I am a product manager at a tech startup. I need to prepare a presentation for our investors about our new AI-powered product. Please help me by:\n\n1. **Summarizing** the current state of AI in the healthcare industry (200 words max).\n2. **Generating** 5 key value propositions for our product (bullet points).\n3. **Analyzing** two potential risks (using a pros/cons table).\n4. **Providing** a rough slide outline for a 10-slide presentation (list each slide's title and key content).\n5. **Writing** a short, compelling closing statement (2 sentences) that I can use to end my presentation.\n\nThroughout your response, ensure your tone is professional yet approachable. Use appropriate formatting (headings, lists, tables) to make it easy to scan. The response should not exceed 1500 words.",
|
||||
"response": "Hello! As a fellow professional, I know how high the stakes are when presenting to investors. You want to demonstrate not just technological brilliance, but also commercial viability and a clear understanding of the landscape.\nHere is a structured toolkit to help you build a compelling, investor-ready presentation.\n\n1. The Landscape: AI in Healthcare (2024–2025)\nThe healthcare industry is currently undergoing a paradigm shift, moving from reactive care to predictive, personalized medicine driven by Artificial Intelligence. We have transitioned from the \"experimental\" phase to the \"integration\" phase. \nCurrently, AI is delivering massive value in three primary domains: Diagnostics, where machine learning models assist radiologists in detecting anomalies with superhuman precision; Drug Discovery, where generative AI is collapsing the timeline for identifying viable molecular candidates from years to months; and Operational Efficiency, where NLP (Natural Language Processing) is being used to automate clinical documentation, drastically reducing provider burnout. \nHowever, the industry is at a critical inflection point. While the potential is immense, the focus has shifted from \"what can AI do?\" to \"how can AI be implemented safely, ethically, and within existing regulatory frameworks (like HIPAA)?\" Investors are no longer looking for general AI; they are looking for specialized, vertically integrated solutions that solve specific clinical or administrative bottlenecks while maintaining high standards of data privacy and accuracy.\n\n2. Key Value Propositions\nSince I don't have your specific product specs, I have drafted these to reflect the high-impact drivers investors look for in medical AI:\n\nEnhanced Clinical Accuracy: Minimizes human error by providing real-time, data-driven decision support for practitioners.\nOperational Cost Reduction: Automates high-volume, low-complexity administrative tasks, allowing specialized staff to focus on patient care.\nAccelerated Time-to-Insight: Processes massive datasets (genomic, imaging, or EHR) in seconds to identify patterns that would take humans weeks to uncover.\nScalable Patient Monitoring: Enables proactive care through continuous data analysis, identifying deteriorating patient conditions before they become emergencies.\nSeamless Workflow Integration: Designed to sit within existing clinical ecosystems (EHRs), ensuring high adoption rates with minimal friction for medical staff.\n\n\n3. Risk Analysis\nInvestors will test your awareness of the \"walls\" your product might hit. Addressing these proactively builds immense credibility.\n\n\n\nRisk Factor\nPotential Pros (Mitigation/Opportunity)\nPotential Cons (Threats)\n\n\n\nRegulatory Compliance & Data Privacy\nAchieving HIPAA/GDPR compliance creates a \"moat\" that makes it harder for smaller, unregulated competitors to enter.\nHigh cost of legal/compliance audits; risk of heavy fines or being barred from certain markets if a breach occurs.\n\n\nAlgorithmic Bias & Clinical Trust\nDeveloping \"Explainable AI\" (XAI) builds deep trust with doctors and sets a new industry standard for transparency.\nIf the AI produces biased results due to skewed training data, it can lead to poor patient outcomes and catastrophic brand damage.\n\n\n\n4. 10-Slide Presentation Outline\n\nSlide 1: Title Slide – Product name, your name/title, and a high-impact tagline.\nSlide 2: The Problem – Define the specific pain point in healthcare (e.g., clinician burnout or diagnostic delays) using a compelling statistic.\nSlide 3: The Solution – Introduce your product as the hero of the story. High-level overview of how it works.\nSlide 4: Market Opportunity – The \"TAM/SAM/SOM\" (Total, Serviceable, and Obtainable market). Show the massive scale of the problem you are solving.\nSlide 5: Product Deep Dive (The \"Magic\") – A visual walkthrough or demo snippet of your core AI functionality.\nSlide 6: Value Proposition & Benefits – Use the bullet points from section 2 here to show the ROI for hospitals/clinics.\nSlide 7: Competitive Landscape – A quadrant map or checklist showing why you are superior to current \"legacy\" methods or other AI startups.\nSlide 8: Business Model – How you make money (e.g., SaaS subscription per seat, per patient, or per scan).\nSlide 9: Traction & Roadmap – Where you are now (pilots, partnerships, or revenue) and where you are going in the next 18 months.\nSlide 10: The Team & The Ask – Why you are the right people to win, and the specific amount of capital you are raising and what it will be used for.\n\n\n5. The Closing Statement\n\"We aren't just building a new piece of software; we are building the intelligent foundation for the next generation of human health. We invite you to join us in turning the promise of AI into the reality of better patient outcomes.\"\n",
|
||||
"response_preview": "Hello! As a fellow professional, I know how high the stakes are when presenting to investors. You want to demonstrate not just technological brilliance, but also commercial viability and a clear understanding of the landscape.\nHere is a structured toolkit to help you build a compelling, investor-rea",
|
||||
"elapsed_sec": 51.98,
|
||||
"screenshot_path": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA06_evidence.png",
|
||||
"screenshot": "C:\\Users\\ZIAD\\OneDrive\\سطح المكتب\\ghaymah_solidpoint.ai\\SolidPoint_Security_Framework\\2_LLM_QA_Evaluation\\evidence\\screenshots\\QA06_evidence.png",
|
||||
"metrics": {
|
||||
"response_time_sec": 51.98,
|
||||
"word_count": 711,
|
||||
"has_bold": false,
|
||||
"has_list": true,
|
||||
"has_table": false,
|
||||
"readability_score": 69.1,
|
||||
"formatting_score": 50,
|
||||
"accuracy_score": 56,
|
||||
"overall_score": 53
|
||||
},
|
||||
"evaluation": "FAIL",
|
||||
"eval_reason": "UX issues: Missing bold text, Missing table format",
|
||||
"notes": "UX issues: Missing bold text, Missing table format",
|
||||
"confidence": 0.53,
|
||||
"passed": false,
|
||||
"timestamp": "2026-08-27T16:05:35.640499+00:00"
|
||||
}
|
||||
]
|
||||
ثنائية
2_LLM_QA_Evaluation/src/__pycache__/qa_client.cpython-313.pyc
Normal file
ثنائية
2_LLM_QA_Evaluation/src/__pycache__/qa_client.cpython-313.pyc
Normal file
ملف ثنائي غير معروض.
636
2_LLM_QA_Evaluation/src/qa_client.py
Normal file
636
2_LLM_QA_Evaluation/src/qa_client.py
Normal file
@@ -0,0 +1,636 @@
|
||||
"""
|
||||
qa_client.py - LLM Functional Testing & QA Client
|
||||
===================================================
|
||||
Playwright-based automation for running QA test cases against
|
||||
SolidPoint OS (https://os.solidpoint.ai).
|
||||
|
||||
Sends each QA test prompt, captures full-page screenshots,
|
||||
evaluates responses using heuristic checks, and outputs qa_results.json.
|
||||
|
||||
Usage:
|
||||
python qa_client.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logging
|
||||
# ---------------------------------------------------------------------------
|
||||
LOG_DIR = Path(__file__).parent.parent / "logs"
|
||||
LOG_DIR.mkdir(exist_ok=True)
|
||||
|
||||
_fmt = logging.Formatter(
|
||||
"[%(asctime)s] %(levelname)-7s %(name)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
_sh = logging.StreamHandler(sys.stderr)
|
||||
_sh.setFormatter(_fmt)
|
||||
|
||||
_fh = logging.FileHandler(LOG_DIR / "qa_client.log", encoding="utf-8")
|
||||
_fh.setFormatter(_fmt)
|
||||
|
||||
log = logging.getLogger("qa.client")
|
||||
log.setLevel(logging.DEBUG)
|
||||
log.addHandler(_sh)
|
||||
log.addHandler(_fh)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paths
|
||||
# ---------------------------------------------------------------------------
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
PROJECT_ROOT = BASE_DIR.parent # SolidPoint_Security_Framework/
|
||||
ENV_FILE = PROJECT_ROOT / "config" / ".env"
|
||||
TESTCASES_JSON = BASE_DIR / "data" / "qa_testcases.json"
|
||||
EVIDENCE_DIR = BASE_DIR / "evidence" / "screenshots"
|
||||
EVIDENCE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
RESULTS_DIR = BASE_DIR / "reports"
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
RESULTS_FILE = RESULTS_DIR / "qa_results.json"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Selectors (same SPA as V2 red-team client)
|
||||
# ---------------------------------------------------------------------------
|
||||
SEL_TEXTAREA = ".input-row textarea"
|
||||
SEL_SEND_BTN = ".send-btn"
|
||||
SEL_MSG_BODY = ".m-body"
|
||||
SEL_TYPING = ".typing"
|
||||
SEL_WELCOME = ".welcome"
|
||||
SEL_SIDEBAR = ".sidebar"
|
||||
SEL_CONTINUE_BTN = 'button:has-text("Continue"), button:has-text("continue")'
|
||||
SEL_OVERLAY = ".sk-overlay, .upgrade-overlay"
|
||||
SEL_OVERLAY_CLOSE = ".sk-btns button, .sk-overlay .close, .upgrade-cancel"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config loaders
|
||||
# ---------------------------------------------------------------------------
|
||||
def load_env() -> dict:
|
||||
"""Parse .env file into a dict."""
|
||||
env = {}
|
||||
if not ENV_FILE.exists():
|
||||
log.warning(".env not found at %s", ENV_FILE)
|
||||
return env
|
||||
for line in ENV_FILE.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
env[k.strip()] = v.strip()
|
||||
return env
|
||||
|
||||
|
||||
def load_testcases() -> list[dict]:
|
||||
"""Load QA test cases from JSON."""
|
||||
if not TESTCASES_JSON.exists():
|
||||
log.error("qa_testcases.json not found at %s", TESTCASES_JSON)
|
||||
return []
|
||||
with open(TESTCASES_JSON, encoding="utf-8") as f:
|
||||
cases = json.load(f)
|
||||
log.info("Loaded %d QA test cases", len(cases))
|
||||
return cases
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Browser helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
def login_with_token(page, token: str) -> bool:
|
||||
"""Inject PRO_CHAT_TOKEN into localStorage and reload."""
|
||||
log.info("Injecting token via localStorage...")
|
||||
for attempt in range(3):
|
||||
try:
|
||||
page.goto("https://os.solidpoint.ai", wait_until="load", timeout=60_000)
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt == 2:
|
||||
raise e
|
||||
log.warning("goto failed: %s, retrying...", e)
|
||||
page.wait_for_timeout(3000)
|
||||
|
||||
page.evaluate(f"localStorage.setItem('chat_token', '{token}')")
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
page.reload(wait_until="load", timeout=60_000)
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt == 2:
|
||||
raise e
|
||||
log.warning("reload failed: %s, retrying...", e)
|
||||
page.wait_for_timeout(3000)
|
||||
|
||||
page.wait_for_timeout(3000)
|
||||
success = (
|
||||
page.locator(SEL_SIDEBAR).is_visible()
|
||||
or page.locator(SEL_WELCOME).is_visible()
|
||||
or page.locator(SEL_TEXTAREA).is_visible()
|
||||
)
|
||||
if success:
|
||||
log.info("Token-based auth succeeded")
|
||||
else:
|
||||
log.warning("Token-based auth may have failed")
|
||||
return success
|
||||
|
||||
|
||||
def dismiss_overlay(page):
|
||||
"""Dismiss any overlay/popup that blocks interaction (e.g., upgrade prompts)."""
|
||||
try:
|
||||
overlay = page.locator(SEL_OVERLAY)
|
||||
if overlay.first.is_visible(timeout=1000):
|
||||
log.debug("Overlay detected, attempting to dismiss...")
|
||||
# Try clicking close/cancel buttons inside the overlay
|
||||
close_btns = page.locator(SEL_OVERLAY_CLOSE)
|
||||
for i in range(close_btns.count()):
|
||||
btn = close_btns.nth(i)
|
||||
if btn.is_visible():
|
||||
btn_text = (btn.text_content() or "").strip().lower()
|
||||
# Click cancel/close/skip/not now buttons, avoid upgrade buttons
|
||||
if any(w in btn_text for w in ["cancel", "close", "skip", "not now", "later", "no", "dismiss"]):
|
||||
btn.click()
|
||||
page.wait_for_timeout(1000)
|
||||
log.debug("Dismissed overlay via '%s' button", btn_text)
|
||||
return True
|
||||
# If no obvious cancel button, try clicking the last button (often cancel)
|
||||
if close_btns.count() > 0:
|
||||
last_btn = close_btns.nth(close_btns.count() - 1)
|
||||
if last_btn.is_visible():
|
||||
last_btn.click()
|
||||
page.wait_for_timeout(1000)
|
||||
log.debug("Dismissed overlay via last button")
|
||||
return True
|
||||
# Last resort: click outside the overlay or press Escape
|
||||
page.keyboard.press("Escape")
|
||||
page.wait_for_timeout(1000)
|
||||
log.debug("Dismissed overlay via Escape key")
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Force remove any upgrade overlays if clicking didn't work
|
||||
page.evaluate("document.querySelectorAll('.upgrade-overlay, .sk-overlay').forEach(e => e.remove())")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def clear_chat_state(page):
|
||||
"""Create a new chat session by clicking New Chat or reloading."""
|
||||
dismiss_overlay(page)
|
||||
try:
|
||||
new_chat = page.locator('button:has-text("New"), .new-chat-btn, .new-chat')
|
||||
if new_chat.first.is_visible(timeout=2000):
|
||||
new_chat.first.click()
|
||||
page.wait_for_timeout(2000)
|
||||
dismiss_overlay(page)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
page.reload(wait_until="load", timeout=30_000)
|
||||
page.wait_for_timeout(3000)
|
||||
dismiss_overlay(page)
|
||||
|
||||
|
||||
def send_message(page, text: str, timeout_sec: int = 120) -> str:
|
||||
"""Type a message, click send, wait for the response, return it."""
|
||||
# Dismiss any overlay first
|
||||
dismiss_overlay(page)
|
||||
|
||||
textarea = page.locator(SEL_TEXTAREA).first
|
||||
try:
|
||||
textarea.click(timeout=5000)
|
||||
except Exception:
|
||||
# Overlay might have reappeared
|
||||
dismiss_overlay(page)
|
||||
page.wait_for_timeout(1000)
|
||||
textarea.click(timeout=10000)
|
||||
textarea.fill(text)
|
||||
page.wait_for_timeout(500)
|
||||
|
||||
send_btn = page.locator(SEL_SEND_BTN).first
|
||||
try:
|
||||
send_btn.click(timeout=3000)
|
||||
except Exception:
|
||||
# Force click
|
||||
page.evaluate("document.querySelectorAll('.upgrade-overlay, .sk-overlay').forEach(e => e.remove())")
|
||||
page.wait_for_timeout(500)
|
||||
send_btn.click(force=True)
|
||||
|
||||
log.debug("Message sent, waiting for response...")
|
||||
|
||||
# Wait for response stability
|
||||
last_content = ""
|
||||
stable_count = 0
|
||||
deadline = time.time() + timeout_sec
|
||||
|
||||
while time.time() < deadline:
|
||||
dismiss_overlay(page)
|
||||
|
||||
# Check for Continue button
|
||||
try:
|
||||
cont = page.locator(SEL_CONTINUE_BTN).first
|
||||
if cont.is_visible(timeout=500):
|
||||
cont.click()
|
||||
log.debug("Clicked Continue button")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
bodies = page.locator(SEL_MSG_BODY).all()
|
||||
current_content = bodies[-1].text_content() if bodies else ""
|
||||
|
||||
is_typing = page.locator(SEL_TYPING).first.is_visible()
|
||||
|
||||
if current_content == last_content and current_content != "" and not is_typing:
|
||||
stable_count += 1
|
||||
else:
|
||||
stable_count = 0
|
||||
|
||||
last_content = current_content
|
||||
|
||||
if stable_count >= 3:
|
||||
log.debug("Response stabilized")
|
||||
return current_content
|
||||
|
||||
page.wait_for_timeout(1000)
|
||||
|
||||
log.warning("Timed out waiting for response to stabilize")
|
||||
return last_content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Evaluation functions
|
||||
# ---------------------------------------------------------------------------
|
||||
def evaluate_keyword_presence(response: str, eval_config: dict) -> dict:
|
||||
"""Check if response contains required keywords from source."""
|
||||
keywords = eval_config.get("required_keywords", [])
|
||||
min_required = eval_config.get("min_keywords", 3)
|
||||
|
||||
found = []
|
||||
missing = []
|
||||
for kw in keywords:
|
||||
if kw.lower() in response.lower():
|
||||
found.append(kw)
|
||||
else:
|
||||
missing.append(kw)
|
||||
|
||||
passed = len(found) >= min_required
|
||||
return {
|
||||
"passed": passed,
|
||||
"evaluation": "PASS" if passed else "FAIL",
|
||||
"confidence": round(len(found) / max(len(keywords), 1), 2),
|
||||
"found_keywords": found,
|
||||
"missing_keywords": missing,
|
||||
"reason": f"Found {len(found)}/{len(keywords)} keywords (min: {min_required})"
|
||||
}
|
||||
|
||||
|
||||
def evaluate_json_schema(response: str, eval_config: dict) -> dict:
|
||||
"""Validate JSON structure and schema."""
|
||||
# Try to extract JSON from response (may be wrapped in markdown)
|
||||
json_text = response.strip()
|
||||
|
||||
# Strip markdown code blocks if present
|
||||
json_match = re.search(r'```(?:json)?\s*\n?([\s\S]*?)\n?```', json_text)
|
||||
if json_match:
|
||||
json_text = json_match.group(1).strip()
|
||||
|
||||
# Also try to find raw JSON array
|
||||
if not json_text.startswith("["):
|
||||
arr_match = re.search(r'(\[[\s\S]*\])', json_text)
|
||||
if arr_match:
|
||||
json_text = arr_match.group(1)
|
||||
|
||||
try:
|
||||
data = json.loads(json_text)
|
||||
except json.JSONDecodeError as e:
|
||||
return {
|
||||
"passed": False,
|
||||
"evaluation": "FAIL",
|
||||
"confidence": 0.0,
|
||||
"reason": f"Invalid JSON: {e}",
|
||||
"parsed_data": None
|
||||
}
|
||||
|
||||
errors = []
|
||||
|
||||
# Check type
|
||||
expected_type = eval_config.get("expected_type", "array")
|
||||
if expected_type == "array" and not isinstance(data, list):
|
||||
errors.append(f"Expected array, got {type(data).__name__}")
|
||||
|
||||
# Check length
|
||||
expected_len = eval_config.get("expected_length")
|
||||
if expected_len and isinstance(data, list) and len(data) != expected_len:
|
||||
errors.append(f"Expected {expected_len} items, got {len(data)}")
|
||||
|
||||
# Check fields
|
||||
required_fields = eval_config.get("required_fields", [])
|
||||
field_types = eval_config.get("field_types", {})
|
||||
valid_roles = eval_config.get("valid_roles", [])
|
||||
|
||||
if isinstance(data, list):
|
||||
for i, item in enumerate(data):
|
||||
if not isinstance(item, dict):
|
||||
errors.append(f"Item {i} is not an object")
|
||||
continue
|
||||
for field in required_fields:
|
||||
if field not in item:
|
||||
errors.append(f"Item {i} missing field '{field}'")
|
||||
for field, expected_ftype in field_types.items():
|
||||
if field in item:
|
||||
val = item[field]
|
||||
if expected_ftype == "str" and not isinstance(val, str):
|
||||
errors.append(f"Item {i}.{field}: expected str, got {type(val).__name__}")
|
||||
elif expected_ftype == "bool" and not isinstance(val, bool):
|
||||
errors.append(f"Item {i}.{field}: expected bool, got {type(val).__name__}")
|
||||
if valid_roles and "role" in item:
|
||||
if item["role"] not in valid_roles:
|
||||
errors.append(f"Item {i}.role: '{item['role']}' not in {valid_roles}")
|
||||
|
||||
passed = len(errors) == 0
|
||||
return {
|
||||
"passed": passed,
|
||||
"evaluation": "PASS" if passed else "FAIL",
|
||||
"confidence": 1.0 if passed else max(0, 1 - len(errors) * 0.2),
|
||||
"reason": "Valid JSON with correct schema" if passed else f"Schema errors: {'; '.join(errors[:5])}",
|
||||
"errors": errors,
|
||||
"parsed_data": data if passed else None
|
||||
}
|
||||
|
||||
|
||||
def evaluate_abstention(response: str, eval_config: dict) -> dict:
|
||||
"""Check if LLM abstains vs hallucinates about fabricated event."""
|
||||
response_lower = response.lower()
|
||||
|
||||
abstention_indicators = eval_config.get("abstention_indicators", [])
|
||||
hallucination_indicators = eval_config.get("hallucination_indicators", [])
|
||||
|
||||
abstention_hits = [ind for ind in abstention_indicators if ind.lower() in response_lower]
|
||||
hallucination_hits = [ind for ind in hallucination_indicators if ind.lower() in response_lower]
|
||||
|
||||
if abstention_hits and not hallucination_hits:
|
||||
return {
|
||||
"passed": True,
|
||||
"evaluation": "PASS",
|
||||
"confidence": 0.9,
|
||||
"reason": f"LLM correctly abstained ({len(abstention_hits)} indicators)",
|
||||
"abstention_indicators_found": abstention_hits,
|
||||
"hallucination_indicators_found": []
|
||||
}
|
||||
elif hallucination_hits:
|
||||
return {
|
||||
"passed": False,
|
||||
"evaluation": "FAIL",
|
||||
"confidence": 0.85,
|
||||
"reason": f"LLM hallucinated ({len(hallucination_hits)} indicators)",
|
||||
"abstention_indicators_found": abstention_hits,
|
||||
"hallucination_indicators_found": hallucination_hits
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"passed": False,
|
||||
"evaluation": "INCONCLUSIVE",
|
||||
"confidence": 0.4,
|
||||
"reason": "No clear abstention or hallucination indicators found",
|
||||
"abstention_indicators_found": [],
|
||||
"hallucination_indicators_found": []
|
||||
}
|
||||
|
||||
|
||||
def calculate_ux_metrics(response: str, elapsed_sec: float) -> dict:
|
||||
"""Calculate basic UX metrics like word count, formatting presence, and readability."""
|
||||
words = len(response.split())
|
||||
has_bold = "**" in response
|
||||
has_list = bool(re.search(r'(?m)^(\s*[-*]|\s*\d+\.) ', response))
|
||||
has_table = "|---" in response or "| ---" in response
|
||||
|
||||
# Simple readability heuristic
|
||||
sentences = max(1, len(re.split(r'[.!?]+', response)))
|
||||
readability_score = min(100, max(0, 100 - (words / sentences) * 2))
|
||||
|
||||
return {
|
||||
"response_time_sec": elapsed_sec,
|
||||
"word_count": words,
|
||||
"has_bold": has_bold,
|
||||
"has_list": has_list,
|
||||
"has_table": has_table,
|
||||
"readability_score": round(readability_score, 1)
|
||||
}
|
||||
|
||||
def evaluate_ux(response: str, eval_config: dict, elapsed_sec: float) -> dict:
|
||||
"""Evaluate UX and formatting based on config."""
|
||||
metrics = calculate_ux_metrics(response, elapsed_sec)
|
||||
|
||||
formatting_score = 100
|
||||
errors = []
|
||||
|
||||
if eval_config.get("requires_bold") and not metrics["has_bold"]:
|
||||
formatting_score -= 20
|
||||
errors.append("Missing bold text")
|
||||
if eval_config.get("requires_list") and not metrics["has_list"]:
|
||||
formatting_score -= 20
|
||||
errors.append("Missing list format")
|
||||
if eval_config.get("requires_table") and not metrics["has_table"]:
|
||||
formatting_score -= 30
|
||||
errors.append("Missing table format")
|
||||
|
||||
min_words = eval_config.get("min_words", 0)
|
||||
if min_words > 0 and metrics["word_count"] < min_words:
|
||||
formatting_score -= 20
|
||||
errors.append(f"Too short ({metrics['word_count']} < {min_words})")
|
||||
|
||||
formatting_score = max(0, formatting_score)
|
||||
passed = formatting_score >= 80
|
||||
|
||||
accuracy_score = min(100, formatting_score + int(metrics["readability_score"] / 10))
|
||||
overall_score = int((formatting_score + accuracy_score) / 2)
|
||||
|
||||
metrics["formatting_score"] = formatting_score
|
||||
metrics["accuracy_score"] = accuracy_score
|
||||
metrics["overall_score"] = overall_score
|
||||
|
||||
return {
|
||||
"passed": passed,
|
||||
"evaluation": "PASS" if passed else "FAIL",
|
||||
"confidence": overall_score / 100.0,
|
||||
"reason": "Good formatting and UX" if passed else f"UX issues: {', '.join(errors)}",
|
||||
"metrics": metrics
|
||||
}
|
||||
|
||||
|
||||
def evaluate_response(response: str, testcase: dict, elapsed_sec: float) -> dict:
|
||||
"""Route to the appropriate evaluator based on test type."""
|
||||
eval_config = testcase.get("evaluation", {})
|
||||
eval_type = eval_config.get("type", "")
|
||||
|
||||
if eval_type == "keyword_presence":
|
||||
base = evaluate_keyword_presence(response, eval_config)
|
||||
base["metrics"] = calculate_ux_metrics(response, elapsed_sec)
|
||||
base["metrics"]["formatting_score"] = 100
|
||||
base["metrics"]["accuracy_score"] = int(base["confidence"] * 100)
|
||||
base["metrics"]["overall_score"] = int((100 + base["metrics"]["accuracy_score"]) / 2)
|
||||
return base
|
||||
elif eval_type == "json_schema":
|
||||
base = evaluate_json_schema(response, eval_config)
|
||||
base["metrics"] = calculate_ux_metrics(response, elapsed_sec)
|
||||
base["metrics"]["formatting_score"] = int(base["confidence"] * 100)
|
||||
base["metrics"]["accuracy_score"] = int(base["confidence"] * 100)
|
||||
base["metrics"]["overall_score"] = int(base["confidence"] * 100)
|
||||
return base
|
||||
elif eval_type == "abstention_check":
|
||||
base = evaluate_abstention(response, eval_config)
|
||||
base["metrics"] = calculate_ux_metrics(response, elapsed_sec)
|
||||
base["metrics"]["formatting_score"] = 100
|
||||
base["metrics"]["accuracy_score"] = int(base["confidence"] * 100)
|
||||
base["metrics"]["overall_score"] = int((100 + base["metrics"]["accuracy_score"]) / 2)
|
||||
return base
|
||||
elif eval_type == "ux_evaluation":
|
||||
return evaluate_ux(response, eval_config, elapsed_sec)
|
||||
else:
|
||||
return {
|
||||
"passed": False,
|
||||
"evaluation": "UNKNOWN",
|
||||
"confidence": 0.0,
|
||||
"reason": f"Unknown evaluation type: {eval_type}",
|
||||
"metrics": calculate_ux_metrics(response, elapsed_sec)
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main suite runner
|
||||
# ---------------------------------------------------------------------------
|
||||
def run_qa_suite():
|
||||
"""Execute all QA test cases."""
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
env = load_env()
|
||||
token = env.get("PRO_CHAT_TOKEN", "")
|
||||
if not token:
|
||||
log.error("PRO_CHAT_TOKEN not found in .env")
|
||||
return
|
||||
|
||||
testcases = load_testcases()
|
||||
if not testcases:
|
||||
log.error("No test cases loaded")
|
||||
return
|
||||
|
||||
results = []
|
||||
print(f"Starting QA suite with {len(testcases)} tests...")
|
||||
log.info("=" * 60)
|
||||
log.info("STARTING LLM QA EVALUATION SUITE")
|
||||
log.info("Target: https://os.solidpoint.ai | Tests: %d", len(testcases))
|
||||
log.info("=" * 60)
|
||||
|
||||
with sync_playwright() as pw:
|
||||
browser = pw.chromium.launch(headless=True)
|
||||
context = browser.new_context(viewport={"width": 1920, "height": 1080})
|
||||
page = context.new_page()
|
||||
|
||||
# Auth
|
||||
if not login_with_token(page, token):
|
||||
log.error("Authentication failed, aborting")
|
||||
browser.close()
|
||||
return
|
||||
|
||||
for idx, tc in enumerate(testcases):
|
||||
tc_id = tc["id"]
|
||||
tc_title = tc["title"]
|
||||
log.info("--- Test %d/%d [%s] ---", idx + 1, len(testcases), tc_id)
|
||||
log.info("Title: %s", tc_title)
|
||||
|
||||
start = time.time()
|
||||
|
||||
try:
|
||||
# Clear chat state for each test
|
||||
clear_chat_state(page)
|
||||
|
||||
# Send prompt
|
||||
response = send_message(page, tc["message"], timeout_sec=120)
|
||||
elapsed = round(time.time() - start, 2)
|
||||
|
||||
# Screenshot
|
||||
screenshot_path = EVIDENCE_DIR / f"{tc_id}_evidence.png"
|
||||
page.screenshot(path=str(screenshot_path), full_page=True)
|
||||
log.info("Screenshot saved: %s", screenshot_path)
|
||||
|
||||
# Evaluate
|
||||
eval_result = evaluate_response(response, tc, elapsed)
|
||||
|
||||
result = {
|
||||
"id": tc_id,
|
||||
"test_id": tc_id,
|
||||
"cat": tc["cat"],
|
||||
"title": tc_title,
|
||||
"message": tc["message"][:200],
|
||||
"prompt": tc["message"],
|
||||
"response": response,
|
||||
"response_preview": response[:300] if response else "[NO RESPONSE]",
|
||||
"elapsed_sec": elapsed,
|
||||
"screenshot_path": str(screenshot_path),
|
||||
"screenshot": str(screenshot_path),
|
||||
"metrics": eval_result.get("metrics", {}),
|
||||
"evaluation": eval_result["evaluation"],
|
||||
"eval_reason": eval_result["reason"],
|
||||
"notes": eval_result["reason"],
|
||||
"confidence": eval_result.get("confidence", 0),
|
||||
"passed": eval_result["passed"],
|
||||
"timestamp": datetime.now(timezone.utc).isoformat()
|
||||
}
|
||||
|
||||
results.append(result)
|
||||
status_icon = "✅" if eval_result["passed"] else "❌"
|
||||
log.info("[%s] %s %s: %s (%.1fs)",
|
||||
eval_result["evaluation"], status_icon, tc_id,
|
||||
eval_result["reason"], elapsed)
|
||||
|
||||
except Exception as e:
|
||||
elapsed = round(time.time() - start, 2)
|
||||
log.error("Test %s FAILED with exception: %s", tc_id, e)
|
||||
results.append({
|
||||
"id": tc_id,
|
||||
"cat": tc["cat"],
|
||||
"title": tc_title,
|
||||
"message": tc["message"][:200],
|
||||
"response": f"[ERROR: {e}]",
|
||||
"response_preview": f"[ERROR: {e}]",
|
||||
"elapsed_sec": elapsed,
|
||||
"screenshot_path": "",
|
||||
"evaluation": "ERROR",
|
||||
"eval_reason": str(e),
|
||||
"confidence": 0,
|
||||
"passed": False,
|
||||
"eval_details": {},
|
||||
"timestamp": datetime.now(timezone.utc).isoformat()
|
||||
})
|
||||
|
||||
browser.close()
|
||||
|
||||
# Save results
|
||||
RESULTS_FILE.write_text(
|
||||
json.dumps(results, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8"
|
||||
)
|
||||
log.info("Results saved to %s", RESULTS_FILE)
|
||||
|
||||
# Print summary
|
||||
passed = sum(1 for r in results if r["passed"])
|
||||
failed = sum(1 for r in results if not r["passed"])
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"QA SUITE COMPLETE: {passed} passed, {failed} failed out of {len(results)} tests")
|
||||
print(f"Results: {RESULTS_FILE}")
|
||||
print(f"{'=' * 60}")
|
||||
for r in results:
|
||||
icon = "✅" if r["passed"] else "❌"
|
||||
print(f" {icon} {r['id']}: [{r['evaluation']}] {r['eval_reason']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_qa_suite()
|
||||
المرجع في مشكلة جديدة
حظر مستخدم