From fe7860fabe258e91d46b6a889c104cb386b0d848 Mon Sep 17 00:00:00 2001
From: noora_coder <noora@gitpasha.com>
Date: Wed, 5 Aug 2026 09:58:16 +0000
Subject: [PATCH] =?UTF-8?q?=D8=A5=D8=B6=D8=A7=D9=81=D8=A9=20preprocess.py?=
 =?UTF-8?q?=20-=20=D8=A8=D8=B3=D9=85=20=D8=A7=D9=84=D9=84=D9=87=20=D8=A7?=
 =?UTF-8?q?=D9=84=D8=B1=D8=AD=D9=85=D9=86=20=D8=A7=D9=84=D8=B1=D8=AD=D9=8A?=
 =?UTF-8?q?=D9=85?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 preprocess.py | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 preprocess.py

diff --git a/preprocess.py b/preprocess.py
new file mode 100644
index 0000000..195be4d
--- /dev/null
+++ b/preprocess.py
@@ -0,0 +1,42 @@
+"""Arabic NLP Preprocessing Tools"""
+import re
+
+# Arabic diacritics (علامات التشكيل)
+TASHKEEL = re.compile(r"[ً-ْٰ]")
+# Arabic punctuation
+PUNCT = re.compile(r"[،؛؟٪-٭]")
+# Arabic stop words (كلمات التوقف)
+STOP_WORDS = {
+    "في", "من", "على", "إلى", "عن", "كان", "هذا", "هذه",
+    "ذلك", "تلك", "الذي", "التي", "مع", "بعد", "قبل", "حتى",
+    "و", "ثم", "أو", "لا", "ما", "إن", "أن", "كل", "بعض"
+}
+
+def remove_tashkeel(text: str) -> str:
+    """إزالة علامات التشكيل من النص العربي"""
+    return TASHKEEL.sub("", text)
+
+def remove_punctuation(text: str) -> str:
+    """إزالة علامات الترقيم العربية"""
+    return PUNCT.sub("", text)
+
+def tokenize(text: str) -> list:
+    """تجزئة النص إلى كلمات"""
+    return text.split()
+
+def remove_stop_words(tokens: list) -> list:
+    """إزالة كلمات التوقف"""
+    return [t for t in tokens if t not in STOP_WORDS]
+
+def clean_arabic(text: str) -> str:
+    """تنظيف النص العربي بالكامل"""
+    text = remove_tashkeel(text)
+    text = remove_punctuation(text)
+    tokens = tokenize(text)
+    tokens = remove_stop_words(tokens)
+    return " ".join(tokens)
+
+if __name__ == "__main__":
+    sample = "السَّلَامُ عَلَيْكُمْ وَرَحْمَةُ اللَّهِ وَبَرَكَاتُهُ"
+    print(f"Original: {sample}")
+    print(f"Clean:    {clean_arabic(sample)}")