43 أسطر
1.5 KiB
Python
43 أسطر
1.5 KiB
Python
"""Arabic NLP Preprocessing Tools"""
|
||
import re
|
||
|
||
# Arabic diacritics (علامات التشكيل)
|
||
TASHKEEL = re.compile(r"[ً-ْٰ]")
|
||
# Arabic punctuation
|
||
PUNCT = re.compile(r"[،؛؟٪-٭]")
|
||
# Arabic stop words (كلمات التوقف)
|
||
STOP_WORDS = {
|
||
"في", "من", "على", "إلى", "عن", "كان", "هذا", "هذه",
|
||
"ذلك", "تلك", "الذي", "التي", "مع", "بعد", "قبل", "حتى",
|
||
"و", "ثم", "أو", "لا", "ما", "إن", "أن", "كل", "بعض"
|
||
}
|
||
|
||
def remove_tashkeel(text: str) -> str:
|
||
"""إزالة علامات التشكيل من النص العربي"""
|
||
return TASHKEEL.sub("", text)
|
||
|
||
def remove_punctuation(text: str) -> str:
|
||
"""إزالة علامات الترقيم العربية"""
|
||
return PUNCT.sub("", text)
|
||
|
||
def tokenize(text: str) -> list:
|
||
"""تجزئة النص إلى كلمات"""
|
||
return text.split()
|
||
|
||
def remove_stop_words(tokens: list) -> list:
|
||
"""إزالة كلمات التوقف"""
|
||
return [t for t in tokens if t not in STOP_WORDS]
|
||
|
||
def clean_arabic(text: str) -> str:
|
||
"""تنظيف النص العربي بالكامل"""
|
||
text = remove_tashkeel(text)
|
||
text = remove_punctuation(text)
|
||
tokens = tokenize(text)
|
||
tokens = remove_stop_words(tokens)
|
||
return " ".join(tokens)
|
||
|
||
if __name__ == "__main__":
|
||
sample = "السَّلَامُ عَلَيْكُمْ وَرَحْمَةُ اللَّهِ وَبَرَكَاتُهُ"
|
||
print(f"Original: {sample}")
|
||
print(f"Clean: {clean_arabic(sample)}")
|