mi
54 lines
1.6 KiB
Python
54 lines
1.6 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import unicodedata
|
|
from dataclasses import dataclass
|
|
|
|
_BIDI = {"RLE", "LRE", "RLO", "LRO", "PDF", "RLI", "LRI", "FSI", "PDI"}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class NormalizedText:
|
|
display: str
|
|
analysis: str
|
|
analysis_sha256: bytes
|
|
flags: tuple[str, ...]
|
|
|
|
|
|
def normalize_text(raw: str) -> NormalizedText:
|
|
display = unicodedata.normalize("NFKC", raw.replace("\r\n", "\n").replace("\r", "\n"))
|
|
if len(display) > 10_000:
|
|
raise ValueError("text exceeds 10000 normalized code points")
|
|
flags: set[str] = set()
|
|
analysis: list[str] = []
|
|
scripts: set[str] = set()
|
|
for char in display:
|
|
category = unicodedata.category(char)
|
|
bidi = unicodedata.bidirectional(char)
|
|
name = unicodedata.name(char, "")
|
|
if bidi in _BIDI:
|
|
flags.add("bidi_control")
|
|
continue
|
|
if category == "Cf":
|
|
flags.add("default_ignorable")
|
|
if char in {"\u200b", "\u200c", "\u200d", "\ufeff"}:
|
|
flags.add("zero_width")
|
|
continue
|
|
if char.isspace():
|
|
analysis.append(" " if char != "\n" else "\n")
|
|
else:
|
|
analysis.append(char)
|
|
if "LATIN" in name:
|
|
scripts.add("latin")
|
|
elif "CYRILLIC" in name:
|
|
scripts.add("cyrillic")
|
|
if len(scripts) > 1:
|
|
flags.add("mixed_script")
|
|
analysis_form = "".join(analysis)
|
|
return NormalizedText(
|
|
display=display,
|
|
analysis=analysis_form,
|
|
analysis_sha256=hashlib.sha256(analysis_form.encode()).digest(),
|
|
flags=tuple(sorted(flags)),
|
|
)
|