diff --git a/preprocess.py b/preprocess.py index 195be4d..1a87b07 100644 --- a/preprocess.py +++ b/preprocess.py @@ -1,42 +1,9 @@ -"""Arabic NLP Preprocessing Tools""" import re - -# Arabic diacritics (علامات التشكيل) -TASHKEEL = re.compile(r"[ً-ْٰ]") -# Arabic punctuation -PUNCT = re.compile(r"[،؛؟٪-٭]") -# Arabic stop words (كلمات التوقف) -STOP_WORDS = { - "في", "من", "على", "إلى", "عن", "كان", "هذا", "هذه", - "ذلك", "تلك", "الذي", "التي", "مع", "بعد", "قبل", "حتى", - "و", "ثم", "أو", "لا", "ما", "إن", "أن", "كل", "بعض" -} - -def remove_tashkeel(text: str) -> str: - """إزالة علامات التشكيل من النص العربي""" - return TASHKEEL.sub("", text) - -def remove_punctuation(text: str) -> str: - """إزالة علامات الترقيم العربية""" - return PUNCT.sub("", text) - -def tokenize(text: str) -> list: - """تجزئة النص إلى كلمات""" - return text.split() - -def remove_stop_words(tokens: list) -> list: - """إزالة كلمات التوقف""" - return [t for t in tokens if t not in STOP_WORDS] - -def clean_arabic(text: str) -> str: - """تنظيف النص العربي بالكامل""" - text = remove_tashkeel(text) - text = remove_punctuation(text) - tokens = tokenize(text) - tokens = remove_stop_words(tokens) - return " ".join(tokens) - -if __name__ == "__main__": - sample = "السَّلَامُ عَلَيْكُمْ وَرَحْمَةُ اللَّهِ وَبَرَكَاتُهُ" - print(f"Original: {sample}") - print(f"Clean: {clean_arabic(sample)}") +TASHKEEL=re.compile(r'[\u064b-\u0652\u0670]') +STOP={'\u0641\u064a','\u0645\u0646','\u0639\u0644\u0649','\u0625\u0644\u0649','\u0639\u0646','\u0643\u0627\u0646','\u0647\u0630\u0627','\u0648','\u0644\u0627','\u0645\u0627','\u0643\u0644'} +def remove_tashkeel(t): return TASHKEEL.sub('',t) +def tokenize(t): return t.split() +def remove_stop_words(ts): return [t for t in ts if t not in STOP] +def clean_arabic(t): + t=remove_tashkeel(t) + return ' '.join(remove_stop_words(tokenize(t)))