Update preprocess.py

preprocess.py

@@ -1,42 +1,9 @@
-"""Arabic NLP Preprocessing Tools"""
 import re
-
-# Arabic diacritics (علامات التشكيل)
-TASHKEEL = re.compile(r"[ً-ْٰ]")
-# Arabic punctuation
-PUNCT = re.compile(r"[،؛؟٪-٭]")
-# Arabic stop words (كلمات التوقف)
-STOP_WORDS = {
-    "في", "من", "على", "إلى", "عن", "كان", "هذا", "هذه",
-    "ذلك", "تلك", "الذي", "التي", "مع", "بعد", "قبل", "حتى",
-    "و", "ثم", "أو", "لا", "ما", "إن", "أن", "كل", "بعض"
-}
-
-def remove_tashkeel(text: str) -> str:
-    """إزالة علامات التشكيل من النص العربي"""
-    return TASHKEEL.sub("", text)
-
-def remove_punctuation(text: str) -> str:
-    """إزالة علامات الترقيم العربية"""
-    return PUNCT.sub("", text)
-
-def tokenize(text: str) -> list:
-    """تجزئة النص إلى كلمات"""
-    return text.split()
-
-def remove_stop_words(tokens: list) -> list:
-    """إزالة كلمات التوقف"""
-    return [t for t in tokens if t not in STOP_WORDS]
-
-def clean_arabic(text: str) -> str:
-    """تنظيف النص العربي بالكامل"""
-    text = remove_tashkeel(text)
-    text = remove_punctuation(text)
-    tokens = tokenize(text)
-    tokens = remove_stop_words(tokens)
-    return " ".join(tokens)
-
-if __name__ == "__main__":
-    sample = "السَّلَامُ عَلَيْكُمْ وَرَحْمَةُ اللَّهِ وَبَرَكَاتُهُ"
-    print(f"Original: {sample}")
-    print(f"Clean:    {clean_arabic(sample)}")
+TASHKEEL=re.compile(r'[\u064b-\u0652\u0670]')
+STOP={'\u0641\u064a','\u0645\u0646','\u0639\u0644\u0649','\u0625\u0644\u0649','\u0639\u0646','\u0643\u0627\u0646','\u0647\u0630\u0627','\u0648','\u0644\u0627','\u0645\u0627','\u0643\u0644'}
+def remove_tashkeel(t): return TASHKEEL.sub('',t)
+def tokenize(t): return t.split()
+def remove_stop_words(ts): return [t for t in ts if t not in STOP]
+def clean_arabic(t):
+    t=remove_tashkeel(t)
+    return ' '.join(remove_stop_words(tokenize(t)))