Проект разделен на два репозитория

2026-08-14 15:42:45 +03:00
parent e06a77ee1d
commit bbef7a30c9
521 changed files with 2597 additions and 2302 deletions
@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import hashlib
+import unicodedata
+from dataclasses import dataclass
+
+_BIDI = {"RLE", "LRE", "RLO", "LRO", "PDF", "RLI", "LRI", "FSI", "PDI"}
+
+
+@dataclass(frozen=True)
+class NormalizedText:
+    display: str
+    analysis: str
+    analysis_sha256: bytes
+    flags: tuple[str, ...]
+
+
+def normalize_text(raw: str) -> NormalizedText:
+    display = unicodedata.normalize("NFKC", raw.replace("\r\n", "\n").replace("\r", "\n"))
+    if len(display) > 10_000:
+        raise ValueError("text exceeds 10000 normalized code points")
+    flags: set[str] = set()
+    analysis: list[str] = []
+    scripts: set[str] = set()
+    for char in display:
+        category = unicodedata.category(char)
+        bidi = unicodedata.bidirectional(char)
+        name = unicodedata.name(char, "")
+        if bidi in _BIDI:
+            flags.add("bidi_control")
+            continue
+        if category == "Cf":
+            flags.add("default_ignorable")
+            if char in {"\u200b", "\u200c", "\u200d", "\ufeff"}:
+                flags.add("zero_width")
+            continue
+        if char.isspace():
+            analysis.append(" " if char != "\n" else "\n")
+        else:
+            analysis.append(char)
+        if "LATIN" in name:
+            scripts.add("latin")
+        elif "CYRILLIC" in name:
+            scripts.add("cyrillic")
+    if len(scripts) > 1:
+        flags.add("mixed_script")
+    analysis_form = "".join(analysis)
+    return NormalizedText(
+        display=display,
+        analysis=analysis_form,
+        analysis_sha256=hashlib.sha256(analysis_form.encode()).digest(),
+        flags=tuple(sorted(flags)),
+    )