diff --git a/scrapping_moad_gov_sy_.ipynb b/scrapping_moad_gov_sy_.ipynb new file mode 100644 index 0000000..e799fb7 --- /dev/null +++ b/scrapping_moad_gov_sy_.ipynb @@ -0,0 +1,1182 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "code", + "source": [ + " pip install playwright\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "bHoQ_qAoZQxv", + "outputId": "c16286ef-f2cc-4da9-ba98-b789ac807027" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting playwright\n", + " Downloading playwright-1.58.0-py3-none-manylinux1_x86_64.whl.metadata (3.5 kB)\n", + "Requirement already satisfied: pyee<14,>=13 in /usr/local/lib/python3.12/dist-packages (from playwright) (13.0.1)\n", + "Requirement already satisfied: greenlet<4.0.0,>=3.1.1 in /usr/local/lib/python3.12/dist-packages (from playwright) (3.3.2)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.12/dist-packages (from pyee<14,>=13->playwright) (4.15.0)\n", + "Downloading playwright-1.58.0-py3-none-manylinux1_x86_64.whl (46.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.2/46.2 MB\u001b[0m \u001b[31m18.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: playwright\n", + "Successfully installed playwright-1.58.0\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + " pip install browserforge" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Av4ZOChdZQkl", + "outputId": "fcc0ea3c-64ae-46ab-c18e-53bf4573092d" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting browserforge\n", + " Downloading browserforge-1.2.4-py3-none-any.whl.metadata (28 kB)\n", + "Collecting apify_fingerprint_datapoints (from browserforge)\n", + " Downloading apify_fingerprint_datapoints-0.11.0-py3-none-any.whl.metadata (14 kB)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.12/dist-packages (from browserforge) (8.3.1)\n", + "Downloading browserforge-1.2.4-py3-none-any.whl (37 kB)\n", + "Downloading apify_fingerprint_datapoints-0.11.0-py3-none-any.whl (726 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m726.5/726.5 kB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: apify_fingerprint_datapoints, browserforge\n", + "Successfully installed apify_fingerprint_datapoints-0.11.0 browserforge-1.2.4\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "pip install msgspec\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "L97-H0pYZnrq", + "outputId": "d9c91b31-34e5-4b8e-9d99-ac37a168e247" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting msgspec\n", + " Downloading msgspec-0.20.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (5.5 kB)\n", + "Downloading msgspec-0.20.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (224 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.9/224.9 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: msgspec\n", + "Successfully installed msgspec-0.20.0\n" + ] + } + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UsWWLQbtY0Hk" + }, + "outputs": [], + "source": [ + "from scrapling.spiders import Spider, Request, Response\n", + "from scrapling.fetchers import FetcherSession, AsyncStealthySession\n", + "\n", + "class MultiSessionSpider(Spider):\n", + " name = \"multi\"\n", + " start_urls = [\"http://www.mohe.gov.sy/\"]\n", + "\n", + " def configure_sessions(self, manager):\n", + " manager.add(\"fast\", FetcherSession(impersonate=\"chrome\"))\n", + " manager.add(\"stealth\", AsyncStealthySession(headless=True), lazy=True)\n", + "\n", + " async def parse(self, response: Response):\n", + " for link in response.css('a::attr(href)').getall():\n", + " # Route protected pages through the stealth session\n", + " if \"protected\" in link:\n", + " yield Request(link, sid=\"stealth\")\n", + " else:\n", + " yield Request(link, sid=\"fast\", callback=self.parse) # explicit callback" + ] + }, + { + "cell_type": "code", + "source": [ + "QuotesSpider(crawldir=\"./crawl_data\").start()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 141 + }, + "id": "w1TBvsDYZ0ck", + "outputId": "3dc790a9-8ea3-4ad0-e5d7-e146ac1fb27a" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "error", + "ename": "NameError", + "evalue": "name 'QuotesSpider' is not defined", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipython-input-3259/1167272156.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mQuotesSpider\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcrawldir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"./crawl_data\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'QuotesSpider' is not defined" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "pip install patchright" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Mu5Viw53ZMs4", + "outputId": "ea160ad6-3f70-4108-cf1d-8b86e3a001aa" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting patchright\n", + " Downloading patchright-1.58.0-py3-none-manylinux1_x86_64.whl.metadata (13 kB)\n", + "Collecting pyee<14,>=13 (from patchright)\n", + " Downloading pyee-13.0.1-py3-none-any.whl.metadata (3.0 kB)\n", + "Requirement already satisfied: greenlet<4.0.0,>=3.1.1 in /usr/local/lib/python3.12/dist-packages (from patchright) (3.3.2)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.12/dist-packages (from pyee<14,>=13->patchright) (4.15.0)\n", + "Downloading patchright-1.58.0-py3-none-manylinux1_x86_64.whl (46.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.2/46.2 MB\u001b[0m \u001b[31m18.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading pyee-13.0.1-py3-none-any.whl (15 kB)\n", + "Installing collected packages: pyee, patchright\n", + "Successfully installed patchright-1.58.0 pyee-13.0.1\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "pip install scrapling" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "dmAsRfslY0oB", + "outputId": "6c1e6b75-d841-42fd-ffeb-977db94178de" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting scrapling\n", + " Downloading scrapling-0.4.1-py3-none-any.whl.metadata (29 kB)\n", + "Requirement already satisfied: lxml>=6.0.2 in /usr/local/lib/python3.12/dist-packages (from scrapling) (6.0.2)\n", + "Collecting cssselect>=1.4.0 (from scrapling)\n", + " Downloading cssselect-1.4.0-py3-none-any.whl.metadata (2.4 kB)\n", + "Requirement already satisfied: orjson>=3.11.7 in /usr/local/lib/python3.12/dist-packages (from scrapling) (3.11.7)\n", + "Collecting tld>=0.13.1 (from scrapling)\n", + " Downloading tld-0.13.1-py2.py3-none-any.whl.metadata (10 kB)\n", + "Collecting w3lib>=2.4.0 (from scrapling)\n", + " Downloading w3lib-2.4.0-py3-none-any.whl.metadata (2.3 kB)\n", + "Requirement already satisfied: typing_extensions in /usr/local/lib/python3.12/dist-packages (from scrapling) (4.15.0)\n", + "Downloading scrapling-0.4.1-py3-none-any.whl (116 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.1/116.1 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading cssselect-1.4.0-py3-none-any.whl (18 kB)\n", + "Downloading tld-0.13.1-py2.py3-none-any.whl (274 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m274.7/274.7 kB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading w3lib-2.4.0-py3-none-any.whl (21 kB)\n", + "Installing collected packages: w3lib, tld, cssselect, scrapling\n", + "Successfully installed cssselect-1.4.0 scrapling-0.4.1 tld-0.13.1 w3lib-2.4.0\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "6q__VUv7ZDBa" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "from scrapling.fetchers import Fetcher\n", + "\n", + "# Simple one-off HTTP fetch\n", + "page = Fetcher.get(\"https://moad.gov.sy/\")\n", + "\n", + "# Extract text with CSS\n", + "quotes = page.css(\".quote .text::text\")\n", + "print(quotes)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "90wYg2hHaR0Y", + "outputId": "8e00b0df-e8c8-4669-b083-16411200b293" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "[2026-03-02 12:05:37] INFO: Fetched (200) (referer: https://www.google.com/search?q=moad)\n", + "INFO:scrapling:Fetched (200) (referer: https://www.google.com/search?q=moad)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[]\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "from scrapling.fetchers import Fetcher\n", + "\n", + "page = Fetcher.get(\"https://moad.gov.sy/\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "nWJ5vevFaYy8", + "outputId": "962aec8f-2971-4750-e2b5-23ce3a4cfbd0" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "[2026-03-02 12:05:47] INFO: Fetched (200) (referer: https://www.google.com/search?q=moad)\n", + "INFO:scrapling:Fetched (200) (referer: https://www.google.com/search?q=moad)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "from scrapling.fetchers import FetcherSession\n", + "\n", + "with FetcherSession(impersonate=\"chrome\") as session:\n", + " page = session.get(\"https://moad.gov.sy/\")\n", + " print(page.css(\".quote .text::text\"))" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "dnK1R7tIajuV", + "outputId": "e776de2d-d640-4c85-8813-8ef1d861e70d" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "[2026-03-02 12:05:49] INFO: Fetched (200) (referer: https://www.google.com/search?q=moad)\n", + "INFO:scrapling:Fetched (200) (referer: https://www.google.com/search?q=moad)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[]\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# CSS selectors\n", + "items = page.css(\".item\")\n", + "\n", + "# XPath selectors\n", + "titles = page.xpath('//h1/text()')\n", + "\n", + "# Find by text\n", + "divs = page.find_by_text(\"Example\")\n", + "\n", + "# First matched element\n", + "response = Fetcher.get(\"https://moad.gov.sy/\")\n", + "items = response.css(\".item\")\n", + "first = items[0] if items else None" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4PyQxFzna7nM", + "outputId": "2a5455cf-680d-4316-b828-b00211600c2d" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "[2026-03-02 12:18:52] INFO: Fetched (200) (referer: https://www.google.com/search?q=moad)\n", + "INFO:scrapling:Fetched (200) (referer: https://www.google.com/search?q=moad)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "import asyncio\n", + "from scrapling.fetchers import AsyncStealthySession\n", + "\n", + "url = \"https://moad.gov.sy/\"\n", + "\n", + "async def fetch_one():\n", + " async with AsyncStealthySession() as session:\n", + " response = await session.fetch(url)\n", + " return response\n", + "\n", + "response = asyncio.run(fetch_one())\n", + "\n", + "page = response.selector\n", + "print(page.title)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 231 + }, + "id": "fWg0oy-Ra-ii", + "outputId": "cb288567-c0c0-43d2-983c-e5c29ee13810" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "[2026-03-02 12:28:29] INFO: Fetched (200) (referer: https://www.google.com/search?q=moad)\n", + "INFO:scrapling:Fetched (200) (referer: https://www.google.com/search?q=moad)\n" + ] + }, + { + "output_type": "error", + "ename": "AttributeError", + "evalue": "'Response' object has no attribute 'title'", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipython-input-3259/3424203136.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"https://moad.gov.sy/\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mpage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m: 'Response' object has no attribute 'title'" + ] + } + ] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "hyPlVtONhxB8" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "vq08HIcnhxNe" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "pip install playwright" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "JaWUq9Wkc90L", + "outputId": "d6798484-bd6a-4114-f4cf-ba7386e69b7d" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: playwright in /usr/local/lib/python3.12/dist-packages (1.58.0)\n", + "Requirement already satisfied: pyee<14,>=13 in /usr/local/lib/python3.12/dist-packages (from playwright) (13.0.1)\n", + "Requirement already satisfied: greenlet<4.0.0,>=3.1.1 in /usr/local/lib/python3.12/dist-packages (from playwright) (3.3.2)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.12/dist-packages (from pyee<14,>=13->playwright) (4.15.0)\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + ":82: RuntimeWarning: coroutine 'fetch_all' was never awaited\n", + "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!playwright install" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "avPc7-zaesR3", + "outputId": "dd01af61-88ff-41a0-aa61-d6320ad0497c" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading Chrome for Testing 145.0.7632.6 (playwright chromium v1208)\u001b[2m from https://cdn.playwright.dev/chrome-for-testing-public/145.0.7632.6/linux64/chrome-linux64.zip\u001b[22m\n", + "(node:9958) [DEP0169] DeprecationWarning: `url.parse()` behavior is not standardized and prone to errors that have security implications. Use the WHATWG URL API instead. CVEs are not issued for `url.parse()` vulnerabilities.\n", + "(Use `node --trace-deprecation ...` to show where the warning was created)\n", + "\u001b[1G167.3 MiB [] 0% 0.0s\u001b[0K\u001b[1G167.3 MiB [] 0% 62.0s\u001b[0K\u001b[1G167.3 MiB [] 0% 35.2s\u001b[0K\u001b[1G167.3 MiB [] 0% 25.2s\u001b[0K\u001b[1G167.3 MiB [] 0% 22.4s\u001b[0K\u001b[1G167.3 MiB [] 0% 21.0s\u001b[0K\u001b[1G167.3 MiB [] 0% 13.3s\u001b[0K\u001b[1G167.3 MiB [] 1% 9.3s\u001b[0K\u001b[1G167.3 MiB [] 1% 7.5s\u001b[0K\u001b[1G167.3 MiB [] 2% 6.4s\u001b[0K\u001b[1G167.3 MiB [] 3% 5.5s\u001b[0K\u001b[1G167.3 MiB [] 3% 5.0s\u001b[0K\u001b[1G167.3 MiB [] 4% 5.0s\u001b[0K\u001b[1G167.3 MiB [] 4% 5.4s\u001b[0K\u001b[1G167.3 MiB [] 4% 4.9s\u001b[0K\u001b[1G167.3 MiB [] 5% 4.6s\u001b[0K\u001b[1G167.3 MiB [] 6% 4.2s\u001b[0K\u001b[1G167.3 MiB [] 7% 3.9s\u001b[0K\u001b[1G167.3 MiB [] 7% 3.7s\u001b[0K\u001b[1G167.3 MiB [] 8% 3.6s\u001b[0K\u001b[1G167.3 MiB [] 9% 3.4s\u001b[0K\u001b[1G167.3 MiB [] 10% 3.2s\u001b[0K\u001b[1G167.3 MiB [] 11% 3.1s\u001b[0K\u001b[1G167.3 MiB [] 12% 3.0s\u001b[0K\u001b[1G167.3 MiB [] 12% 2.9s\u001b[0K\u001b[1G167.3 MiB [] 13% 2.9s\u001b[0K\u001b[1G167.3 MiB [] 13% 2.8s\u001b[0K\u001b[1G167.3 MiB [] 14% 2.8s\u001b[0K\u001b[1G167.3 MiB [] 15% 2.8s\u001b[0K\u001b[1G167.3 MiB [] 16% 2.8s\u001b[0K\u001b[1G167.3 MiB [] 16% 2.7s\u001b[0K\u001b[1G167.3 MiB [] 17% 2.6s\u001b[0K\u001b[1G167.3 MiB [] 18% 2.6s\u001b[0K\u001b[1G167.3 MiB [] 19% 2.6s\u001b[0K\u001b[1G167.3 MiB [] 20% 2.5s\u001b[0K\u001b[1G167.3 MiB [] 21% 2.5s\u001b[0K\u001b[1G167.3 MiB [] 21% 2.4s\u001b[0K\u001b[1G167.3 MiB [] 22% 2.4s\u001b[0K\u001b[1G167.3 MiB [] 23% 2.3s\u001b[0K\u001b[1G167.3 MiB [] 24% 2.2s\u001b[0K\u001b[1G167.3 MiB [] 25% 2.2s\u001b[0K\u001b[1G167.3 MiB [] 26% 2.1s\u001b[0K\u001b[1G167.3 MiB [] 27% 2.0s\u001b[0K\u001b[1G167.3 MiB [] 28% 2.0s\u001b[0K\u001b[1G167.3 MiB [] 29% 2.0s\u001b[0K\u001b[1G167.3 MiB [] 30% 1.9s\u001b[0K\u001b[1G167.3 MiB [] 31% 1.9s\u001b[0K\u001b[1G167.3 MiB [] 32% 1.8s\u001b[0K\u001b[1G167.3 MiB [] 33% 1.8s\u001b[0K\u001b[1G167.3 MiB [] 34% 1.8s\u001b[0K\u001b[1G167.3 MiB [] 35% 1.7s\u001b[0K\u001b[1G167.3 MiB [] 36% 1.7s\u001b[0K\u001b[1G167.3 MiB [] 37% 1.6s\u001b[0K\u001b[1G167.3 MiB [] 38% 1.6s\u001b[0K\u001b[1G167.3 MiB [] 39% 1.6s\u001b[0K\u001b[1G167.3 MiB [] 40% 1.5s\u001b[0K\u001b[1G167.3 MiB [] 41% 1.5s\u001b[0K\u001b[1G167.3 MiB [] 42% 1.5s\u001b[0K\u001b[1G167.3 MiB [] 43% 1.4s\u001b[0K\u001b[1G167.3 MiB [] 44% 1.4s\u001b[0K\u001b[1G167.3 MiB [] 45% 1.4s\u001b[0K\u001b[1G167.3 MiB [] 46% 1.4s\u001b[0K\u001b[1G167.3 MiB [] 47% 1.4s\u001b[0K\u001b[1G167.3 MiB [] 48% 1.3s\u001b[0K\u001b[1G167.3 MiB [] 49% 1.3s\u001b[0K\u001b[1G167.3 MiB [] 50% 1.2s\u001b[0K\u001b[1G167.3 MiB [] 51% 1.2s\u001b[0K\u001b[1G167.3 MiB [] 52% 1.2s\u001b[0K\u001b[1G167.3 MiB [] 53% 1.2s\u001b[0K\u001b[1G167.3 MiB [] 54% 1.1s\u001b[0K\u001b[1G167.3 MiB [] 55% 1.1s\u001b[0K\u001b[1G167.3 MiB [] 56% 1.1s\u001b[0K\u001b[1G167.3 MiB [] 57% 1.1s\u001b[0K\u001b[1G167.3 MiB [] 57% 1.0s\u001b[0K\u001b[1G167.3 MiB [] 58% 1.0s\u001b[0K\u001b[1G167.3 MiB [] 59% 1.0s\u001b[0K\u001b[1G167.3 MiB [] 60% 1.0s\u001b[0K\u001b[1G167.3 MiB [] 61% 0.9s\u001b[0K\u001b[1G167.3 MiB [] 62% 0.9s\u001b[0K\u001b[1G167.3 MiB [] 63% 0.9s\u001b[0K\u001b[1G167.3 MiB [] 64% 0.9s\u001b[0K\u001b[1G167.3 MiB [] 65% 0.8s\u001b[0K\u001b[1G167.3 MiB [] 66% 0.8s\u001b[0K\u001b[1G167.3 MiB [] 67% 0.8s\u001b[0K\u001b[1G167.3 MiB [] 68% 0.8s\u001b[0K\u001b[1G167.3 MiB [] 69% 0.7s\u001b[0K\u001b[1G167.3 MiB [] 70% 0.7s\u001b[0K\u001b[1G167.3 MiB [] 71% 0.7s\u001b[0K\u001b[1G167.3 MiB [] 72% 0.7s\u001b[0K\u001b[1G167.3 MiB [] 73% 0.6s\u001b[0K\u001b[1G167.3 MiB [] 74% 0.6s\u001b[0K\u001b[1G167.3 MiB [] 75% 0.6s\u001b[0K\u001b[1G167.3 MiB [] 76% 0.5s\u001b[0K\u001b[1G167.3 MiB [] 77% 0.5s\u001b[0K\u001b[1G167.3 MiB [] 78% 0.5s\u001b[0K\u001b[1G167.3 MiB [] 79% 0.5s\u001b[0K\u001b[1G167.3 MiB [] 80% 0.5s\u001b[0K\u001b[1G167.3 MiB [] 81% 0.4s\u001b[0K\u001b[1G167.3 MiB [] 82% 0.4s\u001b[0K\u001b[1G167.3 MiB [] 83% 0.4s\u001b[0K\u001b[1G167.3 MiB [] 84% 0.4s\u001b[0K\u001b[1G167.3 MiB [] 85% 0.4s\u001b[0K\u001b[1G167.3 MiB [] 85% 0.3s\u001b[0K\u001b[1G167.3 MiB [] 86% 0.3s\u001b[0K\u001b[1G167.3 MiB [] 87% 0.3s\u001b[0K\u001b[1G167.3 MiB [] 88% 0.3s\u001b[0K\u001b[1G167.3 MiB [] 89% 0.3s\u001b[0K\u001b[1G167.3 MiB [] 90% 0.2s\u001b[0K\u001b[1G167.3 MiB [] 92% 0.2s\u001b[0K\u001b[1G167.3 MiB [] 93% 0.2s\u001b[0K\u001b[1G167.3 MiB [] 94% 0.1s\u001b[0K\u001b[1G167.3 MiB [] 95% 0.1s\u001b[0K\u001b[1G167.3 MiB [] 96% 0.1s\u001b[0K\u001b[1G167.3 MiB [] 97% 0.1s\u001b[0K\u001b[1G167.3 MiB [] 98% 0.0s\u001b[0K\u001b[1G167.3 MiB [] 99% 0.0s\u001b[0K\u001b[1G167.3 MiB [] 100% 0.0s\u001b[0K\n", + "Chrome for Testing 145.0.7632.6 (playwright chromium v1208) downloaded to /root/.cache/ms-playwright/chromium-1208\n", + "Downloading Chrome Headless Shell 145.0.7632.6 (playwright chromium-headless-shell v1208)\u001b[2m from https://cdn.playwright.dev/chrome-for-testing-public/145.0.7632.6/linux64/chrome-headless-shell-linux64.zip\u001b[22m\n", + "(node:10013) [DEP0169] DeprecationWarning: `url.parse()` behavior is not standardized and prone to errors that have security implications. Use the WHATWG URL API instead. CVEs are not issued for `url.parse()` vulnerabilities.\n", + "(Use `node --trace-deprecation ...` to show where the warning was created)\n", + "\u001b[1G110.9 MiB [] 0% 0.0s\u001b[0K\u001b[1G110.9 MiB [] 0% 29.1s\u001b[0K\u001b[1G110.9 MiB [] 0% 9.7s\u001b[0K\u001b[1G110.9 MiB [] 0% 8.0s\u001b[0K\u001b[1G110.9 MiB [] 1% 3.9s\u001b[0K\u001b[1G110.9 MiB [] 2% 2.7s\u001b[0K\u001b[1G110.9 MiB [] 4% 2.3s\u001b[0K\u001b[1G110.9 MiB [] 5% 2.0s\u001b[0K\u001b[1G110.9 MiB [] 5% 2.1s\u001b[0K\u001b[1G110.9 MiB [] 6% 2.3s\u001b[0K\u001b[1G110.9 MiB [] 7% 2.1s\u001b[0K\u001b[1G110.9 MiB [] 8% 1.9s\u001b[0K\u001b[1G110.9 MiB [] 10% 1.8s\u001b[0K\u001b[1G110.9 MiB [] 11% 1.7s\u001b[0K\u001b[1G110.9 MiB [] 12% 1.6s\u001b[0K\u001b[1G110.9 MiB [] 14% 1.5s\u001b[0K\u001b[1G110.9 MiB [] 15% 1.5s\u001b[0K\u001b[1G110.9 MiB [] 16% 1.4s\u001b[0K\u001b[1G110.9 MiB [] 17% 1.3s\u001b[0K\u001b[1G110.9 MiB [] 19% 1.3s\u001b[0K\u001b[1G110.9 MiB [] 20% 1.2s\u001b[0K\u001b[1G110.9 MiB [] 22% 1.2s\u001b[0K\u001b[1G110.9 MiB [] 23% 1.2s\u001b[0K\u001b[1G110.9 MiB [] 25% 1.1s\u001b[0K\u001b[1G110.9 MiB [] 26% 1.1s\u001b[0K\u001b[1G110.9 MiB [] 28% 1.1s\u001b[0K\u001b[1G110.9 MiB [] 29% 1.0s\u001b[0K\u001b[1G110.9 MiB [] 31% 1.0s\u001b[0K\u001b[1G110.9 MiB [] 32% 1.0s\u001b[0K\u001b[1G110.9 MiB [] 33% 0.9s\u001b[0K\u001b[1G110.9 MiB [] 35% 0.9s\u001b[0K\u001b[1G110.9 MiB [] 37% 0.9s\u001b[0K\u001b[1G110.9 MiB [] 38% 0.9s\u001b[0K\u001b[1G110.9 MiB [] 39% 0.8s\u001b[0K\u001b[1G110.9 MiB [] 40% 0.8s\u001b[0K\u001b[1G110.9 MiB [] 41% 0.8s\u001b[0K\u001b[1G110.9 MiB [] 42% 0.8s\u001b[0K\u001b[1G110.9 MiB [] 43% 0.8s\u001b[0K\u001b[1G110.9 MiB [] 44% 0.8s\u001b[0K\u001b[1G110.9 MiB [] 45% 0.8s\u001b[0K\u001b[1G110.9 MiB [] 46% 0.8s\u001b[0K\u001b[1G110.9 MiB [] 47% 0.7s\u001b[0K\u001b[1G110.9 MiB [] 48% 0.7s\u001b[0K\u001b[1G110.9 MiB [] 49% 0.7s\u001b[0K\u001b[1G110.9 MiB [] 50% 0.7s\u001b[0K\u001b[1G110.9 MiB [] 51% 0.7s\u001b[0K\u001b[1G110.9 MiB [] 52% 0.7s\u001b[0K\u001b[1G110.9 MiB [] 53% 0.7s\u001b[0K\u001b[1G110.9 MiB [] 54% 0.7s\u001b[0K\u001b[1G110.9 MiB [] 55% 0.7s\u001b[0K\u001b[1G110.9 MiB [] 56% 0.6s\u001b[0K\u001b[1G110.9 MiB [] 57% 0.6s\u001b[0K\u001b[1G110.9 MiB [] 58% 0.6s\u001b[0K\u001b[1G110.9 MiB [] 59% 0.6s\u001b[0K\u001b[1G110.9 MiB [] 60% 0.6s\u001b[0K\u001b[1G110.9 MiB [] 61% 0.6s\u001b[0K\u001b[1G110.9 MiB [] 62% 0.6s\u001b[0K\u001b[1G110.9 MiB [] 63% 0.6s\u001b[0K\u001b[1G110.9 MiB [] 64% 0.5s\u001b[0K\u001b[1G110.9 MiB [] 65% 0.5s\u001b[0K\u001b[1G110.9 MiB [] 66% 0.5s\u001b[0K\u001b[1G110.9 MiB [] 67% 0.5s\u001b[0K\u001b[1G110.9 MiB [] 68% 0.5s\u001b[0K\u001b[1G110.9 MiB [] 69% 0.5s\u001b[0K\u001b[1G110.9 MiB [] 70% 0.5s\u001b[0K\u001b[1G110.9 MiB [] 71% 0.5s\u001b[0K\u001b[1G110.9 MiB [] 72% 0.4s\u001b[0K\u001b[1G110.9 MiB [] 73% 0.4s\u001b[0K\u001b[1G110.9 MiB [] 74% 0.4s\u001b[0K\u001b[1G110.9 MiB [] 75% 0.4s\u001b[0K\u001b[1G110.9 MiB [] 76% 0.4s\u001b[0K\u001b[1G110.9 MiB [] 77% 0.4s\u001b[0K\u001b[1G110.9 MiB [] 78% 0.3s\u001b[0K\u001b[1G110.9 MiB [] 79% 0.3s\u001b[0K\u001b[1G110.9 MiB [] 80% 0.3s\u001b[0K\u001b[1G110.9 MiB [] 81% 0.3s\u001b[0K\u001b[1G110.9 MiB [] 82% 0.3s\u001b[0K\u001b[1G110.9 MiB [] 83% 0.3s\u001b[0K\u001b[1G110.9 MiB [] 84% 0.3s\u001b[0K\u001b[1G110.9 MiB [] 85% 0.3s\u001b[0K\u001b[1G110.9 MiB [] 85% 0.2s\u001b[0K\u001b[1G110.9 MiB [] 86% 0.2s\u001b[0K\u001b[1G110.9 MiB [] 87% 0.2s\u001b[0K\u001b[1G110.9 MiB [] 88% 0.2s\u001b[0K\u001b[1G110.9 MiB [] 89% 0.2s\u001b[0K\u001b[1G110.9 MiB [] 90% 0.2s\u001b[0K\u001b[1G110.9 MiB [] 92% 0.1s\u001b[0K\u001b[1G110.9 MiB [] 93% 0.1s\u001b[0K\u001b[1G110.9 MiB [] 94% 0.1s\u001b[0K\u001b[1G110.9 MiB [] 95% 0.1s\u001b[0K\u001b[1G110.9 MiB [] 96% 0.1s\u001b[0K\u001b[1G110.9 MiB [] 97% 0.0s\u001b[0K\u001b[1G110.9 MiB [] 98% 0.0s\u001b[0K\u001b[1G110.9 MiB [] 99% 0.0s\u001b[0K\u001b[1G110.9 MiB [] 100% 0.0s\u001b[0K\n", + "Chrome Headless Shell 145.0.7632.6 (playwright chromium-headless-shell v1208) downloaded to /root/.cache/ms-playwright/chromium_headless_shell-1208\n", + "Downloading Firefox 146.0.1 (playwright firefox v1509)\u001b[2m from https://cdn.playwright.dev/dbazure/download/playwright/builds/firefox/1509/firefox-ubuntu-22.04.zip\u001b[22m\n", + "(node:10064) [DEP0169] DeprecationWarning: `url.parse()` behavior is not standardized and prone to errors that have security implications. Use the WHATWG URL API instead. CVEs are not issued for `url.parse()` vulnerabilities.\n", + "(Use `node --trace-deprecation ...` to show where the warning was created)\n", + "\u001b[1G99.5 MiB [] 0% 0.0s\u001b[0K\u001b[1G99.5 MiB [] 0% 14.5s\u001b[0K\u001b[1G99.5 MiB [] 0% 10.9s\u001b[0K\u001b[1G99.5 MiB [] 0% 13.4s\u001b[0K\u001b[1G99.5 MiB [] 0% 16.1s\u001b[0K\u001b[1G99.5 MiB [] 0% 13.5s\u001b[0K\u001b[1G99.5 MiB [] 1% 8.1s\u001b[0K\u001b[1G99.5 MiB [] 1% 6.3s\u001b[0K\u001b[1G99.5 MiB [] 2% 4.7s\u001b[0K\u001b[1G99.5 MiB [] 3% 4.1s\u001b[0K\u001b[1G99.5 MiB [] 4% 3.7s\u001b[0K\u001b[1G99.5 MiB [] 5% 3.5s\u001b[0K\u001b[1G99.5 MiB [] 5% 3.2s\u001b[0K\u001b[1G99.5 MiB [] 6% 3.1s\u001b[0K\u001b[1G99.5 MiB [] 7% 3.1s\u001b[0K\u001b[1G99.5 MiB [] 7% 3.3s\u001b[0K\u001b[1G99.5 MiB [] 8% 3.2s\u001b[0K\u001b[1G99.5 MiB [] 8% 3.3s\u001b[0K\u001b[1G99.5 MiB [] 9% 3.3s\u001b[0K\u001b[1G99.5 MiB [] 10% 3.4s\u001b[0K\u001b[1G99.5 MiB [] 10% 3.5s\u001b[0K\u001b[1G99.5 MiB [] 10% 3.6s\u001b[0K\u001b[1G99.5 MiB [] 10% 3.7s\u001b[0K\u001b[1G99.5 MiB [] 11% 3.8s\u001b[0K\u001b[1G99.5 MiB [] 11% 3.7s\u001b[0K\u001b[1G99.5 MiB [] 12% 3.7s\u001b[0K\u001b[1G99.5 MiB [] 12% 3.6s\u001b[0K\u001b[1G99.5 MiB [] 13% 3.5s\u001b[0K\u001b[1G99.5 MiB [] 14% 3.4s\u001b[0K\u001b[1G99.5 MiB [] 15% 3.4s\u001b[0K\u001b[1G99.5 MiB [] 16% 3.4s\u001b[0K\u001b[1G99.5 MiB [] 17% 3.4s\u001b[0K\u001b[1G99.5 MiB [] 18% 3.2s\u001b[0K\u001b[1G99.5 MiB [] 19% 3.2s\u001b[0K\u001b[1G99.5 MiB [] 20% 3.1s\u001b[0K\u001b[1G99.5 MiB [] 21% 3.0s\u001b[0K\u001b[1G99.5 MiB [] 22% 3.0s\u001b[0K\u001b[1G99.5 MiB [] 23% 2.9s\u001b[0K\u001b[1G99.5 MiB [] 24% 2.9s\u001b[0K\u001b[1G99.5 MiB [] 25% 2.8s\u001b[0K\u001b[1G99.5 MiB [] 26% 2.8s\u001b[0K\u001b[1G99.5 MiB [] 26% 2.7s\u001b[0K\u001b[1G99.5 MiB [] 27% 2.7s\u001b[0K\u001b[1G99.5 MiB [] 28% 2.7s\u001b[0K\u001b[1G99.5 MiB [] 29% 2.6s\u001b[0K\u001b[1G99.5 MiB [] 30% 2.5s\u001b[0K\u001b[1G99.5 MiB [] 31% 2.5s\u001b[0K\u001b[1G99.5 MiB [] 31% 2.4s\u001b[0K\u001b[1G99.5 MiB [] 32% 2.4s\u001b[0K\u001b[1G99.5 MiB [] 33% 2.4s\u001b[0K\u001b[1G99.5 MiB [] 33% 2.3s\u001b[0K\u001b[1G99.5 MiB [] 34% 2.3s\u001b[0K\u001b[1G99.5 MiB [] 35% 2.3s\u001b[0K\u001b[1G99.5 MiB [] 35% 2.4s\u001b[0K\u001b[1G99.5 MiB [] 36% 2.4s\u001b[0K\u001b[1G99.5 MiB [] 36% 2.5s\u001b[0K\u001b[1G99.5 MiB [] 37% 2.5s\u001b[0K\u001b[1G99.5 MiB [] 37% 2.6s\u001b[0K\u001b[1G99.5 MiB [] 38% 2.5s\u001b[0K\u001b[1G99.5 MiB [] 38% 2.6s\u001b[0K\u001b[1G99.5 MiB [] 39% 2.6s\u001b[0K\u001b[1G99.5 MiB [] 39% 2.5s\u001b[0K\u001b[1G99.5 MiB [] 40% 2.5s\u001b[0K\u001b[1G99.5 MiB [] 41% 2.4s\u001b[0K\u001b[1G99.5 MiB [] 42% 2.4s\u001b[0K\u001b[1G99.5 MiB [] 43% 2.3s\u001b[0K\u001b[1G99.5 MiB [] 44% 2.2s\u001b[0K\u001b[1G99.5 MiB [] 45% 2.2s\u001b[0K\u001b[1G99.5 MiB [] 46% 2.1s\u001b[0K\u001b[1G99.5 MiB [] 47% 2.0s\u001b[0K\u001b[1G99.5 MiB [] 48% 2.0s\u001b[0K\u001b[1G99.5 MiB [] 48% 2.1s\u001b[0K\u001b[1G99.5 MiB [] 49% 2.0s\u001b[0K\u001b[1G99.5 MiB [] 50% 2.0s\u001b[0K\u001b[1G99.5 MiB [] 51% 1.9s\u001b[0K\u001b[1G99.5 MiB [] 52% 1.9s\u001b[0K\u001b[1G99.5 MiB [] 52% 1.8s\u001b[0K\u001b[1G99.5 MiB [] 53% 1.8s\u001b[0K\u001b[1G99.5 MiB [] 54% 1.7s\u001b[0K\u001b[1G99.5 MiB [] 55% 1.7s\u001b[0K\u001b[1G99.5 MiB [] 56% 1.7s\u001b[0K\u001b[1G99.5 MiB [] 57% 1.6s\u001b[0K\u001b[1G99.5 MiB [] 58% 1.6s\u001b[0K\u001b[1G99.5 MiB [] 59% 1.5s\u001b[0K\u001b[1G99.5 MiB [] 60% 1.5s\u001b[0K\u001b[1G99.5 MiB [] 61% 1.4s\u001b[0K\u001b[1G99.5 MiB [] 62% 1.4s\u001b[0K\u001b[1G99.5 MiB [] 63% 1.3s\u001b[0K\u001b[1G99.5 MiB [] 65% 1.2s\u001b[0K\u001b[1G99.5 MiB [] 66% 1.2s\u001b[0K\u001b[1G99.5 MiB [] 67% 1.1s\u001b[0K\u001b[1G99.5 MiB [] 69% 1.0s\u001b[0K\u001b[1G99.5 MiB [] 71% 1.0s\u001b[0K\u001b[1G99.5 MiB [] 72% 0.9s\u001b[0K\u001b[1G99.5 MiB [] 73% 0.9s\u001b[0K\u001b[1G99.5 MiB [] 74% 0.9s\u001b[0K\u001b[1G99.5 MiB [] 74% 0.8s\u001b[0K\u001b[1G99.5 MiB [] 75% 0.8s\u001b[0K\u001b[1G99.5 MiB [] 77% 0.8s\u001b[0K\u001b[1G99.5 MiB [] 78% 0.7s\u001b[0K\u001b[1G99.5 MiB [] 79% 0.7s\u001b[0K\u001b[1G99.5 MiB [] 80% 0.6s\u001b[0K\u001b[1G99.5 MiB [] 81% 0.6s\u001b[0K\u001b[1G99.5 MiB [] 82% 0.6s\u001b[0K\u001b[1G99.5 MiB [] 83% 0.5s\u001b[0K\u001b[1G99.5 MiB [] 84% 0.5s\u001b[0K\u001b[1G99.5 MiB [] 85% 0.5s\u001b[0K\u001b[1G99.5 MiB [] 86% 0.4s\u001b[0K\u001b[1G99.5 MiB [] 87% 0.4s\u001b[0K\u001b[1G99.5 MiB [] 89% 0.3s\u001b[0K\u001b[1G99.5 MiB [] 90% 0.3s\u001b[0K\u001b[1G99.5 MiB [] 91% 0.2s\u001b[0K\u001b[1G99.5 MiB [] 92% 0.2s\u001b[0K\u001b[1G99.5 MiB [] 93% 0.2s\u001b[0K\u001b[1G99.5 MiB [] 94% 0.2s\u001b[0K\u001b[1G99.5 MiB [] 96% 0.1s\u001b[0K\u001b[1G99.5 MiB [] 97% 0.1s\u001b[0K\u001b[1G99.5 MiB [] 98% 0.0s\u001b[0K\u001b[1G99.5 MiB [] 99% 0.0s\u001b[0K\u001b[1G99.5 MiB [] 100% 0.0s\u001b[0K\n", + "Firefox 146.0.1 (playwright firefox v1509) downloaded to /root/.cache/ms-playwright/firefox-1509\n", + "Downloading WebKit 26.0 (playwright webkit v2248)\u001b[2m from https://cdn.playwright.dev/dbazure/download/playwright/builds/webkit/2248/webkit-ubuntu-22.04.zip\u001b[22m\n", + "(node:10115) [DEP0169] DeprecationWarning: `url.parse()` behavior is not standardized and prone to errors that have security implications. Use the WHATWG URL API instead. CVEs are not issued for `url.parse()` vulnerabilities.\n", + "(Use `node --trace-deprecation ...` to show where the warning was created)\n", + "\u001b[1G99.4 MiB [] 0% 0.0s\u001b[0K\u001b[1G99.4 MiB [] 0% 15.1s\u001b[0K\u001b[1G99.4 MiB [] 0% 8.9s\u001b[0K\u001b[1G99.4 MiB [] 1% 4.6s\u001b[0K\u001b[1G99.4 MiB [] 1% 3.4s\u001b[0K\u001b[1G99.4 MiB [] 3% 2.6s\u001b[0K\u001b[1G99.4 MiB [] 3% 2.4s\u001b[0K\u001b[1G99.4 MiB [] 5% 2.1s\u001b[0K\u001b[1G99.4 MiB [] 6% 1.9s\u001b[0K\u001b[1G99.4 MiB [] 7% 1.9s\u001b[0K\u001b[1G99.4 MiB [] 8% 1.9s\u001b[0K\u001b[1G99.4 MiB [] 10% 1.7s\u001b[0K\u001b[1G99.4 MiB [] 11% 1.6s\u001b[0K\u001b[1G99.4 MiB [] 12% 1.6s\u001b[0K\u001b[1G99.4 MiB [] 13% 1.5s\u001b[0K\u001b[1G99.4 MiB [] 15% 1.4s\u001b[0K\u001b[1G99.4 MiB [] 16% 1.3s\u001b[0K\u001b[1G99.4 MiB [] 18% 1.3s\u001b[0K\u001b[1G99.4 MiB [] 20% 1.2s\u001b[0K\u001b[1G99.4 MiB [] 22% 1.1s\u001b[0K\u001b[1G99.4 MiB [] 24% 1.1s\u001b[0K\u001b[1G99.4 MiB [] 25% 1.0s\u001b[0K\u001b[1G99.4 MiB [] 27% 1.0s\u001b[0K\u001b[1G99.4 MiB [] 29% 0.9s\u001b[0K\u001b[1G99.4 MiB [] 30% 0.9s\u001b[0K\u001b[1G99.4 MiB [] 32% 0.9s\u001b[0K\u001b[1G99.4 MiB [] 34% 0.8s\u001b[0K\u001b[1G99.4 MiB [] 36% 0.8s\u001b[0K\u001b[1G99.4 MiB [] 38% 0.7s\u001b[0K\u001b[1G99.4 MiB [] 40% 0.7s\u001b[0K\u001b[1G99.4 MiB [] 42% 0.7s\u001b[0K\u001b[1G99.4 MiB [] 43% 0.7s\u001b[0K\u001b[1G99.4 MiB [] 45% 0.6s\u001b[0K\u001b[1G99.4 MiB [] 47% 0.6s\u001b[0K\u001b[1G99.4 MiB [] 48% 0.6s\u001b[0K\u001b[1G99.4 MiB [] 49% 0.6s\u001b[0K\u001b[1G99.4 MiB [] 50% 0.6s\u001b[0K\u001b[1G99.4 MiB [] 52% 0.6s\u001b[0K\u001b[1G99.4 MiB [] 53% 0.6s\u001b[0K\u001b[1G99.4 MiB [] 55% 0.5s\u001b[0K\u001b[1G99.4 MiB [] 56% 0.5s\u001b[0K\u001b[1G99.4 MiB [] 57% 0.5s\u001b[0K\u001b[1G99.4 MiB [] 58% 0.5s\u001b[0K\u001b[1G99.4 MiB [] 60% 0.5s\u001b[0K\u001b[1G99.4 MiB [] 61% 0.5s\u001b[0K\u001b[1G99.4 MiB [] 63% 0.4s\u001b[0K\u001b[1G99.4 MiB [] 66% 0.4s\u001b[0K\u001b[1G99.4 MiB [] 68% 0.4s\u001b[0K\u001b[1G99.4 MiB [] 70% 0.3s\u001b[0K\u001b[1G99.4 MiB [] 71% 0.3s\u001b[0K\u001b[1G99.4 MiB [] 73% 0.3s\u001b[0K\u001b[1G99.4 MiB [] 74% 0.3s\u001b[0K\u001b[1G99.4 MiB [] 75% 0.3s\u001b[0K\u001b[1G99.4 MiB [] 76% 0.3s\u001b[0K\u001b[1G99.4 MiB [] 77% 0.3s\u001b[0K\u001b[1G99.4 MiB [] 78% 0.3s\u001b[0K\u001b[1G99.4 MiB [] 79% 0.3s\u001b[0K\u001b[1G99.4 MiB [] 80% 0.2s\u001b[0K\u001b[1G99.4 MiB [] 81% 0.2s\u001b[0K\u001b[1G99.4 MiB [] 82% 0.2s\u001b[0K\u001b[1G99.4 MiB [] 83% 0.2s\u001b[0K\u001b[1G99.4 MiB [] 84% 0.2s\u001b[0K\u001b[1G99.4 MiB [] 85% 0.2s\u001b[0K\u001b[1G99.4 MiB [] 86% 0.2s\u001b[0K\u001b[1G99.4 MiB [] 87% 0.2s\u001b[0K\u001b[1G99.4 MiB [] 88% 0.2s\u001b[0K\u001b[1G99.4 MiB [] 89% 0.1s\u001b[0K\u001b[1G99.4 MiB [] 90% 0.1s\u001b[0K\u001b[1G99.4 MiB [] 91% 0.1s\u001b[0K\u001b[1G99.4 MiB [] 92% 0.1s\u001b[0K\u001b[1G99.4 MiB [] 93% 0.1s\u001b[0K\u001b[1G99.4 MiB [] 94% 0.1s\u001b[0K\u001b[1G99.4 MiB [] 95% 0.1s\u001b[0K\u001b[1G99.4 MiB [] 96% 0.1s\u001b[0K\u001b[1G99.4 MiB [] 97% 0.0s\u001b[0K\u001b[1G99.4 MiB [] 98% 0.0s\u001b[0K\u001b[1G99.4 MiB [] 99% 0.0s\u001b[0K\u001b[1G99.4 MiB [] 100% 0.0s\u001b[0K\n", + "WebKit 26.0 (playwright webkit v2248) downloaded to /root/.cache/ms-playwright/webkit-2248\n", + "Downloading FFmpeg (playwright ffmpeg v1011)\u001b[2m from https://cdn.playwright.dev/dbazure/download/playwright/builds/ffmpeg/1011/ffmpeg-linux.zip\u001b[22m\n", + "(node:10150) [DEP0169] DeprecationWarning: `url.parse()` behavior is not standardized and prone to errors that have security implications. Use the WHATWG URL API instead. CVEs are not issued for `url.parse()` vulnerabilities.\n", + "(Use `node --trace-deprecation ...` to show where the warning was created)\n", + "\u001b[1G2.3 MiB [] 0% 0.0s\u001b[0K\u001b[1G2.3 MiB [] 5% 0.3s\u001b[0K\u001b[1G2.3 MiB [] 15% 0.2s\u001b[0K\u001b[1G2.3 MiB [] 43% 0.1s\u001b[0K\u001b[1G2.3 MiB [] 69% 0.0s\u001b[0K\u001b[1G2.3 MiB [] 100% 0.0s\u001b[0K\n", + "FFmpeg (playwright ffmpeg v1011) downloaded to /root/.cache/ms-playwright/ffmpeg-1011\n", + "Playwright Host validation warning: \n", + "╔══════════════════════════════════════════════════════╗\n", + "║ Host system is missing dependencies to run browsers. ║\n", + "║ Please install them with the following command: ║\n", + "║ ║\n", + "║ playwright install-deps ║\n", + "║ ║\n", + "║ Alternatively, use apt: ║\n", + "║ apt-get install libxcomposite1\\ ║\n", + "║ libgtk-3-0\\ ║\n", + "║ libatk1.0-0 ║\n", + "║ ║\n", + "║ <3 Playwright Team ║\n", + "╚══════════════════════════════════════════════════════╝\n", + " at validateDependenciesLinux (/usr/local/lib/python3.12/dist-packages/playwright/driver/package/lib/server/registry/dependencies.js:269:9)\n", + "\u001b[90m at process.processTicksAndRejections (node:internal/process/task_queues:103:5)\u001b[39m\n", + " at async Registry._validateHostRequirements (/usr/local/lib/python3.12/dist-packages/playwright/driver/package/lib/server/registry/index.js:991:14)\n", + " at async Registry._validateHostRequirementsForExecutableIfNeeded (/usr/local/lib/python3.12/dist-packages/playwright/driver/package/lib/server/registry/index.js:1113:7)\n", + " at async Registry.validateHostRequirementsForExecutablesIfNeeded (/usr/local/lib/python3.12/dist-packages/playwright/driver/package/lib/server/registry/index.js:1102:7)\n", + " at async i. (/usr/local/lib/python3.12/dist-packages/playwright/driver/package/lib/cli/program.js:174:7)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!apt-get install -y libatk1.0-0" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RvIPwbhie-NZ", + "outputId": "6d570057-a8ef-4d85-d0d5-034f3236bb4f" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Reading package lists... Done\n", + "Building dependency tree... Done\n", + "Reading state information... Done\n", + "The following additional packages will be installed:\n", + " libatk1.0-data\n", + "The following NEW packages will be installed:\n", + " libatk1.0-0 libatk1.0-data\n", + "0 upgraded, 2 newly installed, 0 to remove and 2 not upgraded.\n", + "Need to get 54.8 kB of archives.\n", + "After this operation, 249 kB of additional disk space will be used.\n", + "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 libatk1.0-data all 2.36.0-3build1 [2,824 B]\n", + "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 libatk1.0-0 amd64 2.36.0-3build1 [51.9 kB]\n", + "Fetched 54.8 kB in 1s (86.8 kB/s)\n", + "Selecting previously unselected package libatk1.0-data.\n", + "(Reading database ... 117540 files and directories currently installed.)\n", + "Preparing to unpack .../libatk1.0-data_2.36.0-3build1_all.deb ...\n", + "Unpacking libatk1.0-data (2.36.0-3build1) ...\n", + "Selecting previously unselected package libatk1.0-0:amd64.\n", + "Preparing to unpack .../libatk1.0-0_2.36.0-3build1_amd64.deb ...\n", + "Unpacking libatk1.0-0:amd64 (2.36.0-3build1) ...\n", + "Setting up libatk1.0-data (2.36.0-3build1) ...\n", + "Setting up libatk1.0-0:amd64 (2.36.0-3build1) ...\n", + "Processing triggers for libc-bin (2.35-0ubuntu3.11) ...\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_level_zero_v2.so.0 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_opencl.so.0 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_level_zero.so.0 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libumf.so.1 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtcm.so.1 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtcm_debug.so.1 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libur_loader.so.0 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libhwloc.so.15 is not a symbolic link\n", + "\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!apt-get install -y libatk-bridge2.0-0" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ByWLwKcLfUrC", + "outputId": "03beb5a4-d1e4-4b48-830e-521024673ab8" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Reading package lists... Done\n", + "Building dependency tree... Done\n", + "Reading state information... Done\n", + "The following additional packages will be installed:\n", + " at-spi2-core gsettings-desktop-schemas libatspi2.0-0 libxtst6\n", + " session-migration\n", + "The following NEW packages will be installed:\n", + " at-spi2-core gsettings-desktop-schemas libatk-bridge2.0-0 libatspi2.0-0\n", + " libxtst6 session-migration\n", + "0 upgraded, 6 newly installed, 0 to remove and 2 not upgraded.\n", + "Need to get 256 kB of archives.\n", + "After this operation, 1,215 kB of additional disk space will be used.\n", + "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 libatspi2.0-0 amd64 2.44.0-3 [80.9 kB]\n", + "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxtst6 amd64 2:1.2.3-1build4 [13.4 kB]\n", + "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 session-migration amd64 0.3.6 [9,774 B]\n", + "Get:4 http://archive.ubuntu.com/ubuntu jammy/main amd64 gsettings-desktop-schemas all 42.0-1ubuntu1 [31.1 kB]\n", + "Get:5 http://archive.ubuntu.com/ubuntu jammy/main amd64 at-spi2-core amd64 2.44.0-3 [54.4 kB]\n", + "Get:6 http://archive.ubuntu.com/ubuntu jammy/main amd64 libatk-bridge2.0-0 amd64 2.38.0-3 [66.6 kB]\n", + "Fetched 256 kB in 0s (933 kB/s)\n", + "Selecting previously unselected package libatspi2.0-0:amd64.\n", + "(Reading database ... 117549 files and directories currently installed.)\n", + "Preparing to unpack .../0-libatspi2.0-0_2.44.0-3_amd64.deb ...\n", + "Unpacking libatspi2.0-0:amd64 (2.44.0-3) ...\n", + "Selecting previously unselected package libxtst6:amd64.\n", + "Preparing to unpack .../1-libxtst6_2%3a1.2.3-1build4_amd64.deb ...\n", + "Unpacking libxtst6:amd64 (2:1.2.3-1build4) ...\n", + "Selecting previously unselected package session-migration.\n", + "Preparing to unpack .../2-session-migration_0.3.6_amd64.deb ...\n", + "Unpacking session-migration (0.3.6) ...\n", + "Selecting previously unselected package gsettings-desktop-schemas.\n", + "Preparing to unpack .../3-gsettings-desktop-schemas_42.0-1ubuntu1_all.deb ...\n", + "Unpacking gsettings-desktop-schemas (42.0-1ubuntu1) ...\n", + "Selecting previously unselected package at-spi2-core.\n", + "Preparing to unpack .../4-at-spi2-core_2.44.0-3_amd64.deb ...\n", + "Unpacking at-spi2-core (2.44.0-3) ...\n", + "Selecting previously unselected package libatk-bridge2.0-0:amd64.\n", + "Preparing to unpack .../5-libatk-bridge2.0-0_2.38.0-3_amd64.deb ...\n", + "Unpacking libatk-bridge2.0-0:amd64 (2.38.0-3) ...\n", + "Setting up session-migration (0.3.6) ...\n", + "Created symlink /etc/systemd/user/graphical-session-pre.target.wants/session-migration.service → /usr/lib/systemd/user/session-migration.service.\n", + "Setting up libxtst6:amd64 (2:1.2.3-1build4) ...\n", + "Setting up libatspi2.0-0:amd64 (2.44.0-3) ...\n", + "Setting up libatk-bridge2.0-0:amd64 (2.38.0-3) ...\n", + "Setting up gsettings-desktop-schemas (42.0-1ubuntu1) ...\n", + "Processing triggers for libglib2.0-0:amd64 (2.72.4-0ubuntu2.7) ...\n", + "Processing triggers for libc-bin (2.35-0ubuntu3.11) ...\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_level_zero_v2.so.0 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_opencl.so.0 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libur_adapter_level_zero.so.0 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libumf.so.1 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtcm.so.1 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtcm_debug.so.1 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libur_loader.so.0 is not a symbolic link\n", + "\n", + "/sbin/ldconfig.real: /usr/local/lib/libhwloc.so.15 is not a symbolic link\n", + "\n", + "Processing triggers for man-db (2.10.2-1) ...\n", + "Setting up at-spi2-core (2.44.0-3) ...\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "import requests\n", + "from bs4 import BeautifulSoup\n", + "from urllib.parse import urljoin\n", + "import os\n", + "\n", + "base = \"https://moad.gov.sy/\"\n", + "visited = set()\n", + "\n", + "def scrape_page(url):\n", + " res = requests.get(url)\n", + " soup = BeautifulSoup(res.text, \"html.parser\")\n", + " print(\"Scraping:\", url)\n", + "\n", + " # Ensure the 'pages' directory exists\n", + " os.makedirs('pages', exist_ok=True)\n", + "\n", + " # Save page content\n", + " # Handle the case where url.replace(base, '') might be empty\n", + " filename_segment = url.replace(base, \"\").replace(\"/\", \"_\")\n", + " if not filename_segment:\n", + " filename_segment = \"index\"\n", + " with open(\"pages/\" + filename_segment + \".html\", \"w\") as f:\n", + " f.write(res.text)\n", + "\n", + " # find links\n", + " for a in soup.find_all(\"a\", href=True):\n", + " link = urljoin(base, a[\"href\"])\n", + " if link.startswith(base) and link not in visited:\n", + " visited.add(link)\n", + " scrape_page(link)\n", + "\n", + "scrape_page(base)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "VOQEKlsKhyJx", + "outputId": "cc1cac07-b66c-4d93-f681-8c86caa8d5e9" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Scraping: https://moad.gov.sy/\n", + "Scraping: https://moad.gov.sy/\n", + "Scraping: https://moad.gov.sy/ar/ministry/about\n", + "Scraping: https://moad.gov.sy/ar/ministry/minister\n", + "Scraping: https://moad.gov.sy/ar/ministry/strategic-directions-map\n", + "Scraping: https://moad.gov.sy/ar/legislation/laws\n", + "Scraping: https://moad.gov.sy/ar/legislation/decrees\n", + "Scraping: https://moad.gov.sy/ar/legislation/service-guides\n", + "Scraping: https://moad.gov.sy/news\n", + "Scraping: https://moad.gov.sy/ar/contact/faq\n", + "Scraping: https://moad.gov.sy/ar/contact/important-links\n", + "Scraping: https://moad.gov.sy/ar/ministry\n", + "Scraping: https://moad.gov.sy/ar/legislation\n", + "Scraping: https://moad.gov.sy/ar/news\n", + "Scraping: https://moad.gov.sy/ar/contact\n", + "Scraping: https://moad.gov.sy/contact/faq\n", + "Scraping: https://moad.gov.sy/contact\n", + "Scraping: https://moad.gov.sy/contact/important-links\n", + "Scraping: https://moad.gov.sy/ministry/about\n", + "Scraping: https://moad.gov.sy/ministry\n", + "Scraping: https://moad.gov.sy/ministry/minister\n", + "Scraping: https://moad.gov.sy/legislation\n", + "Scraping: https://moad.gov.sy/legislation/laws\n", + "Scraping: https://moad.gov.sy/legislation/decrees\n", + "Scraping: https://moad.gov.sy/legislation/decrees/42\n", + "Scraping: https://moad.gov.sy/ar/privacy-policy\n", + "Scraping: https://moad.gov.sy/ar/terms-conditions\n", + "Scraping: https://moad.gov.sy/legislation/decrees/43\n", + "Scraping: https://moad.gov.sy/legislation/decrees/46\n", + "Scraping: https://moad.gov.sy/legislation/service-guides\n", + "Scraping: https://moad.gov.sy/legislation/service-guides/building-and-updating-institutional-structures\n", + "Scraping: https://moad.gov.sy/ministry/strategic-directions-map\n", + "Scraping: https://moad.gov.sy/news/bd491579-99a1-448d-a2b4-c2b3d4705c77\n", + "Scraping: https://moad.gov.sy/news/4664cf44-c11f-45e2-bada-2bd7c059900d\n", + "Scraping: https://moad.gov.sy/news/c880be1a-fb29-43f6-8900-9904d3a9f17d\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "import requests\n", + "from bs4 import BeautifulSoup\n", + "import csv\n", + "\n", + "base = \"https://moad.gov.sy\"\n", + "\n", + "news_urls = [\n", + " \"https://moad.gov.sy/news/bd491579-99a1-448d-a2b4-c2b3d4705c77\",\n", + " \"https://moad.gov.sy/news/4664cf44-c11f-45e2-bada-2bd7c059900d\",\n", + " \"https://moad.gov.sy/news/c880be1a-fb29-43f6-8900-9904d3a9f17d\"\n", + "]\n", + "\n", + "with open(\"news_data.csv\", \"w\", newline='', encoding=\"utf-8\") as file:\n", + " writer = csv.writer(file)\n", + " writer.writerow([\"URL\", \"Title\", \"Date\", \"Content\"])\n", + "\n", + " for url in news_urls:\n", + " res = requests.get(url)\n", + " soup = BeautifulSoup(res.text, \"html.parser\")\n", + "\n", + " title = soup.find(\"h1\")\n", + " date = soup.find(class_=\"date\") # inspect actual class name\n", + " content = soup.find(\"div\", class_=\"content\") # inspect actual class name\n", + "\n", + " writer.writerow([\n", + " url,\n", + " title.text.strip() if title else \"\",\n", + " date.text.strip() if date else \"\",\n", + " content.text.strip() if content else \"\"\n", + " ])\n", + "\n", + "print(\"Done.\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "kYQT1q8Vhyl5", + "outputId": "51619263-195a-4dd2-9040-5190038f863d" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Done.\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "03_Pox7ZieT8" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "listing = requests.get(\"https://moad.gov.sy/news\")\n", + "soup = BeautifulSoup(listing.text, \"html.parser\")\n", + "\n", + "links = []\n", + "\n", + "for a in soup.find_all(\"a\", href=True):\n", + " if \"/news/\" in a[\"href\"]:\n", + " full_url = base + a[\"href\"]\n", + " links.append(full_url)\n", + "\n", + "print(links)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "gnxoq5jgimLc", + "outputId": "19b02e8c-506c-462b-a262-ab698045b729" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[]\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "import requests\n", + "\n", + "ra = requests.get(\"https://moad.gov.sy/\")" + ], + "metadata": { + "id": "0OW7Y4eTkfzD" + }, + "execution_count": 4, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "ra.text" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 157 + }, + "id": "8pWLyfZkklW3", + "outputId": "6132632b-6338-476a-d974-34562b03d0aa" + }, + "execution_count": 5, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "'Ministry of Administration Development
دمشق\"دمشق\"
\"التوقيت\"
\"التاريخ\"
\"Telegram\"\"Facebook\"\"LinkedIn\"
\"Logo\"

القائمة

English
\"language

لجنة تمديد الخدمة تعقد اجتماعها الثالث برئاسة وزير التنمية الإدارية

اقرأ المزيد

\"لجنة

فتح باب التسجيل في ماجستير إدارة التحول الرقمي

اقرأ المزيد

\"فتح

دفعة جديدة من قادة التغيير... دبلوم الموارد البشرية

اقرأ المزيد

\"دفعة
رؤية المزيد

الرؤية

منظومة إدارية رائدة ومتميزة، بموارد بشرية مؤهلة وكفاءات تمكن المؤسسات الحكومية في سوريا الجديدة من تحقيق التنمية المستدامة.

الرسالة

قيادة جهود التحديث المؤسسي عبر تطوير أنظمة الخدمة المدنية، وتمكين الكفاءات الوظيفية، وتوظيف التكنولوجيا الإدارية لرفع كفاءة الأداء وتحقيق النماذج والريادة لضمان مؤسسات حكومية فعالة.

آخر الأخبار

رؤية المزيد

تعرف على آخر الأخبار والفعاليات والاعلانات لوزارة الموارد البشرية والتنمية الاجتماعية

\"دعوة

دعوة لزيارة جناح الوزارة في معرض سيريا هايتك 2025

دعماً للتحول الرقمي في المؤسسات، وتماشياً مع الثورة المعلوماتية العصرية،\\nتدعوكم وزارة التنمية الإدارية في الجمهورية العربية السورية لزيارة جناحها الخاص في معرض "سيريا هايتك" لتكنولوجيا المعلومات والاتصالات بنسخته الحادية عشر، المقام في مدينة المعارض بدمشق.\\n\\nوذلك ابتداءً من يوم الخميس الموافق 20/11/2025 حتى يوم الإثنين الموافق 24/11/2025.\\n\\nكونوا معنا، لاكتساب فرصةٍ للتعرف وتبادل الخبرات وبناء شراكاتٍ تنموية تكنولوجية، لتصبحوا جزءاً من البناء التقدمي الحضاري الحديث.

\"فرصة

فرصة إضافية للإبداع: تمديد التقديم في مبادرة أفضل فكرة حكومية

إيماناً بضرورة توفير مساحة أوسع للإبداع الحكومي، ونظراً للطلبات العديدة الواردة من مديريات التنمية الإدارية في الوزارات والمحافظات، وحرصاً من وزارة التنمية الإدارية على إتاحة الفرصة لأكبر عدد من الزملاء لتقديم مقترحاتهم التطويرية…\\n\\nيسرّ الوزارة أن تعلن عن تمديد فترة استقبال الأفكار ضمن مبادرة “أفضل فكرة حكومية” وذلك حتى نهاية الشهر الحالي.\\n\\nتهدف المبادرة إلى تمكين العاملين في مديريات التنمية الإدارية من تقديم أفكار عملية وقابلة للتطبيق، تُسهم في دعم مسارات التحول المؤسسي الشامل ورفع كفاءة الأداء داخل المؤسسات العامة.\\n\\nهذا هو الوقت الأمثل للمشاركة.. فكرتك قد تكون الشرارة الأولى لابتكار مؤسسي حقيقي.\\n\\nالتقديم متاح عبر الرابط:\\nhttps://cloud.moad.gov.sy/apps/forms/s/yG59EFrcxfMGjMLEnyqD5kLm

\"مديرية

مديرية التنمية الإدارية بإدلب تنفذ تدريباً حول اتخاذ القرار وحل المشكلات

نفذت مديرية التنمية الإدارية في إدلب دورة تدريبية بعنوان: "اتخاذ القرار وحل المشكلات"، بهدف تزويد المتدربين بالمهارات والأساليب الفعّالة لاتخاذ القرارات وحل المشكلات بطريقة منطقية ومنظمة.\\n\\nوتناول التدريب محاور متعددة منها: العوامل المؤثرة في عملية اتخاذ القرار، نظريات اتخاذ القرار، إيجابيات وسلبيات القرار، ومراحل اتخاذه، وذلك ضمن جهود المديرية لتعزيز قدرات الكوادر في الإدارة والتخطيط الفعّال.

لمحة عن الوزير

الأستاذ محمد حسان سكاف (مواليد 1990) — وزير التنمية الإدارية، تولى المنصب في 29 مارس 2025.

حاصل على بكالوريوس في الرياضيات التطبيقية وماجستير مهني في الإدارة (امتياز) من جامعة إدلب.

يتميز بخبرة إدارية واسعة بعد شغل عدة مناصب قيادية في حكومة الإنقاذ السورية، ما يؤهله لقيادة عملية تطوير هيكلية وإدارية فعالة.

\"Hero\"
'" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + } + }, + "metadata": {}, + "execution_count": 5 + } + ] + }, + { + "cell_type": "code", + "source": [ + "ra1 = requests.get(\"https://moad.gov.sy/legislation/laws\")" + ], + "metadata": { + "id": "_SMHzkImmRsY" + }, + "execution_count": 6, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "ra1.text" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 157 + }, + "id": "BbQgFo-amZPx", + "outputId": "1c355bbc-1445-49e2-d793-694a00d5ab3d" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "'Ministry of Administration Development'" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + } + }, + "metadata": {}, + "execution_count": 7 + } + ] + }, + { + "cell_type": "code", + "source": [ + "import json\n", + "import requests\n", + "from datetime import datetime\n", + "\n", + "# الرابط المستهدف\n", + "url = \"https://moad.gov.sy/legislation/laws\"\n", + "\n", + "# طلب البيانات\n", + "ra1 = requests.get(url)\n", + "ra1.encoding = 'utf-8' # لضمان قراءة اللغة العربية بشكل صحيح\n", + "\n", + "# تجهيز البيانات للهيكل المطلوب\n", + "data_to_save = {\n", + " \"Url\": url,\n", + " \"Text\": ra1.text,\n", + " \"Timestamp\": datetime.now().isoformat()\n", + "}\n", + "\n", + "# حفظ البيانات في ملف json\n", + "with open('moad_data.json', 'w', encoding='utf-8') as f:\n", + " json.dump(data_to_save, f, ensure_ascii=False, indent=4)\n", + "\n", + "print(\"تم حفظ الملف بنجاح باسم moad_data.json\")" + ], + "metadata": { + "id": "9Qzr701vi1Cm", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "0eece0ac-1437-47a8-f43c-963ed138d80a" + }, + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "تم حفظ الملف بنجاح باسم moad_data.json\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "import requests\n", + "import json\n", + "from datetime import datetime\n", + "import time\n", + "\n", + "# قائمة الروابط التي زودتني بها\n", + "urls = [\n", + " \"https://moad.gov.sy/\",\n", + " \"https://moad.gov.sy/\",\n", + " \"https://moad.gov.sy/ar/ministry/about\",\n", + " \"https://moad.gov.sy/ar/ministry/minister\",\n", + " \"https://moad.gov.sy/ar/ministry/strategic-directions-map\",\n", + " \"https://moad.gov.sy/ar/legislation/laws\",\n", + " \"https://moad.gov.sy/ar/legislation/decrees\",\n", + " \"https://moad.gov.sy/ar/legislation/service-guides\",\n", + " \"https://moad.gov.sy/news\",\n", + " \"https://moad.gov.sy/ar/contact/faq\",\n", + " \"https://moad.gov.sy/ar/contact/important-links\",\n", + " \"https://moad.gov.sy/ar/ministry\",\n", + " \"https://moad.gov.sy/ar/legislation\",\n", + " \"https://moad.gov.sy/ar/news\",\n", + "\" https://moad.gov.sy/ar/contact\",\n", + "\" https://moad.gov.sy/contact/faq\",\n", + " \"https://moad.gov.sy/contact\",\n", + " \"https://moad.gov.sy/contact/important-links\",\n", + " \"https://moad.gov.sy/ministry/about\",\n", + " \"https://moad.gov.sy/ministry\",\n", + " \"https://moad.gov.sy/ministry/minister\",\n", + "\" https://moad.gov.sy/legislation\",\n", + " \"https://moad.gov.sy/legislation/laws\",\n", + " \"https://moad.gov.sy/legislation/decrees\",\n", + " \"https://moad.gov.sy/legislation/decrees/42\",\n", + " \"https://moad.gov.sy/ar/privacy-policy\",\n", + " \"https://moad.gov.sy/ar/terms-conditions\",\n", + " \"https://moad.gov.sy/legislation/decrees/43\",\n", + " \"https://moad.gov.sy/legislation/decrees/46\",\n", + " \"https://moad.gov.sy/legislation/service-guides\",\n", + " \"https://moad.gov.sy/legislation/service-guides/building-and-updating-institutional-structures\",\n", + " \"https://moad.gov.sy/ministry/strategic-directions-map\"\n", + "\n", + "]\n", + "\n", + "# إزالة التكرار من الروابط لضمان عدم سحب نفس الصفحة مرتين\n", + "urls = list(set(urls))\n", + "\n", + "all_data = []\n", + "\n", + "print(f\"جاري بدء عملية السحب لـ {len(urls)} رابط...\")\n", + "\n", + "for url in urls:\n", + " try:\n", + " print(f\"يتم الآن سحب: {url}\")\n", + "\n", + " response = requests.get(url, timeout=10)\n", + " response.encoding = 'utf-8' # لضمان دعم اللغة العربية\n", + "\n", + " if response.status_code == 200:\n", + " entry = {\n", + " \"Url\": url,\n", + " \"Text\": response.text,\n", + " \"Timestamp\": datetime.now().isoformat()\n", + " }\n", + " all_data.append(entry)\n", + " else:\n", + " print(f\"فشل السحب من {url} - رمز الحالة: {response.status_code}\")\n", + "\n", + " # تأخير بسيط (ثانية واحدة) لتجنب حظر السيرفر لعنوانك (IP)\n", + " time.sleep(1)\n", + "\n", + " except Exception as e:\n", + " print(f\"حدث خطأ أثناء سحب {url}: {e}\")\n", + "\n", + "# حفظ كل البيانات في ملف واحد\n", + "with open('moad_full_scrape.json', 'w', encoding='utf-8') as f:\n", + " json.dump(all_data, f, ensure_ascii=False, indent=4)\n", + "\n", + "print(\"-\" * 30)\n", + "print(f\"تمت العملية بنجاح! تم حفظ {len(all_data)} سجل في ملف moad_full_scrape.json\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Vtms8qkqoZpK", + "outputId": "b31ff62f-595c-4d8c-9228-3df40dbcf3ea" + }, + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "جاري بدء عملية السحب لـ 31 رابط...\n", + "يتم الآن سحب: https://moad.gov.sy/ar/contact/important-links\n", + "يتم الآن سحب: https://moad.gov.sy/ar/contact\n", + "يتم الآن سحب: https://moad.gov.sy/ar/ministry/about\n", + "يتم الآن سحب: https://moad.gov.sy/ar/ministry/minister\n", + "يتم الآن سحب: https://moad.gov.sy/ar/contact/faq\n", + "يتم الآن سحب: https://moad.gov.sy/legislation/decrees/42\n", + "يتم الآن سحب: https://moad.gov.sy/legislation/laws\n", + "فشل السحب من https://moad.gov.sy/legislation/laws - رمز الحالة: 500\n", + "يتم الآن سحب: https://moad.gov.sy/ar/legislation\n", + "يتم الآن سحب: https://moad.gov.sy/ar/legislation/decrees\n", + "يتم الآن سحب: https://moad.gov.sy/ar/legislation/laws\n", + "فشل السحب من https://moad.gov.sy/ar/legislation/laws - رمز الحالة: 500\n", + "يتم الآن سحب: https://moad.gov.sy/ministry\n", + "يتم الآن سحب: https://moad.gov.sy/contact/important-links\n", + "يتم الآن سحب: https://moad.gov.sy/ministry/minister\n", + "يتم الآن سحب: https://moad.gov.sy/ar/ministry\n", + "يتم الآن سحب: https://moad.gov.sy/ar/terms-conditions\n", + "يتم الآن سحب: https://moad.gov.sy/ar/news\n", + "فشل السحب من https://moad.gov.sy/ar/news - رمز الحالة: 500\n", + "يتم الآن سحب: https://moad.gov.sy/ar/privacy-policy\n", + "يتم الآن سحب: https://moad.gov.sy/legislation/service-guides/building-and-updating-institutional-structures\n", + "فشل السحب من https://moad.gov.sy/legislation/service-guides/building-and-updating-institutional-structures - رمز الحالة: 500\n", + "يتم الآن سحب: https://moad.gov.sy/ministry/strategic-directions-map\n", + "يتم الآن سحب: https://moad.gov.sy/contact/faq\n", + "يتم الآن سحب: https://moad.gov.sy/contact\n", + "يتم الآن سحب: https://moad.gov.sy/legislation/service-guides\n", + "يتم الآن سحب: https://moad.gov.sy/legislation/decrees\n", + "يتم الآن سحب: https://moad.gov.sy/news\n", + "فشل السحب من https://moad.gov.sy/news - رمز الحالة: 500\n", + "يتم الآن سحب: https://moad.gov.sy/ar/legislation/service-guides\n", + "يتم الآن سحب: https://moad.gov.sy/legislation/decrees/43\n", + "يتم الآن سحب: https://moad.gov.sy/legislation\n", + "يتم الآن سحب: https://moad.gov.sy/ar/ministry/strategic-directions-map\n", + "يتم الآن سحب: https://moad.gov.sy/legislation/decrees/46\n", + "يتم الآن سحب: https://moad.gov.sy/\n", + "يتم الآن سحب: https://moad.gov.sy/ministry/about\n", + "------------------------------\n", + "تمت العملية بنجاح! تم حفظ 26 سجل في ملف moad_full_scrape.json\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "rYpoKbtpqxv2" + }, + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file