Coverage for narrative_harm_classifier/classifier/rules/patterns_loader.py: 97%

1""" 

2classifier/rules/patterns_loader.py — Per-language detection vocabulary. 

3 

4Loads data/patterns/<lang>.yaml files into a LanguagePatterns object with 

5precompiled regexes (explicit precompilation rather than relying on Python's 

6internal re.compile cache — deliberate now that pattern volume grows with 

7each added language and the dog-whistle lexicon). 

8 

9Every file declares a `confidence` of "verified" or "experimental" — see 

10README's Limitations section for what that distinction means in practice. 

11Not every language has negation/counter-speech/benign-context cues defined; 

12missing ones are treated as "no suppression available in this language yet" 

13rather than silently reused from English. 

14""" 

15 

16import re 

17from dataclasses import dataclass, field 

18from functools import lru_cache 

19from pathlib import Path 

20from typing import Optional 

21 

22from narrative_harm_classifier.core.yaml_loader import load_yaml_file 

23 

24DEFAULT_LANGUAGE = "en" 

25 

26 

27def _compile_all(patterns: list[str]) -> list[re.Pattern]: 

28 return [re.compile(p, re.IGNORECASE) for p in patterns] 

29 

30 

31def _compile_dict(d: dict[str, list[str]]) -> dict[str, list[re.Pattern]]: 

32 return {key: _compile_all(patterns) for key, patterns in d.items()} 

33 

34 

35def _combine_and_compile(cue_list: Optional[list[str]]) -> Optional[re.Pattern]: 

36 """Join a YAML list of literal cue phrases into a single compiled alternation.""" 

37 if not cue_list: 

38 return None 

39 alternation = "|".join(re.escape(cue) for cue in cue_list) 

40 return re.compile(rf"\b({alternation})\b", re.IGNORECASE) 

41 

42 

43@dataclass 

44class LanguagePatterns: 

45 language: str 

46 confidence: str # "verified" | "experimental" 

47 identity_anchors: dict[str, list[re.Pattern]] = field(default_factory=dict) 

48 harm_patterns: dict[str, list[re.Pattern]] = field(default_factory=dict) 

49 negation_cues: Optional[re.Pattern] = None 

50 reporting_cues: Optional[re.Pattern] = None 

51 condemnation_cues: Optional[re.Pattern] = None 

52 benign_context_cues: Optional[re.Pattern] = None 

53 obfuscation_map: dict[str, str] = field(default_factory=dict) 

54 

55 def deobfuscate(self, text_lower: str) -> str: 

56 if not self.obfuscation_map: 

57 return text_lower 

58 return text_lower.translate(str.maketrans(self.obfuscation_map)) 

59 

60 

61@lru_cache(maxsize=32) 

62def load_language_patterns(patterns_dir: str, language: str) -> LanguagePatterns: 

63 """ 

64 Load and precompile a language's detection vocabulary. 

65 

66 Raises FileNotFoundError if the language isn't available — callers should 

67 catch this and fall back to DEFAULT_LANGUAGE rather than let a typo 

68 silently produce empty (always-no-harm) patterns. 

69 """ 

70 path = Path(patterns_dir) / f"{language}.yaml" 

71 raw = load_yaml_file(path) 

72 

73 return LanguagePatterns( 

74 language=raw["language"], 

75 confidence=raw["confidence"], 

76 identity_anchors=_compile_dict(raw.get("identity_anchors", {})), 

77 harm_patterns=_compile_dict(raw.get("harm_patterns", {})), 

78 negation_cues=_combine_and_compile(raw.get("negation_cues")), 

79 reporting_cues=_combine_and_compile(raw.get("reporting_cues")), 

80 condemnation_cues=_combine_and_compile(raw.get("condemnation_cues")), 

81 benign_context_cues=_combine_and_compile(raw.get("benign_context_cues")), 

82 obfuscation_map=raw.get("obfuscation_map") or {}, 

83 ) 

84 

85 

86def available_languages(patterns_dir: str) -> list[str]: 

87 return sorted(p.stem for p in Path(patterns_dir).glob("*.yaml"))