Coverage for narrative_harm_classifier/classifier/validators/i18n_smoke.py: 100%

1""" 

2classifier/validators/i18n_smoke.py — Per-language smoke test runner. 

3 

4Deliberately smaller in scope than the English benchmark (see 

5data/i18n_smoke_tests.yaml for why) — this confirms basic detection works 

6per language, broken out so a regression in one specific language is 

7visible rather than hidden in an aggregate. 

8""" 

9 

10from pydantic import BaseModel 

11 

12from narrative_harm_classifier.core.models import ClassifyRequest 

13from narrative_harm_classifier.core.yaml_loader import load_yaml_file 

14from narrative_harm_classifier.classifier.rules.engine import ClassificationEngine 

15 

16 

17class I18nSmokeCaseResult(BaseModel): 

18 language: str 

19 text: str 

20 expected_is_harmful: bool 

21 actual_is_harmful: bool 

22 

23 @property 

24 def passed(self) -> bool: 

25 return self.expected_is_harmful == self.actual_is_harmful 

26 

27 

28class I18nSmokeReport(BaseModel): 

29 total: int 

30 passed: int 

31 failed_cases: list[I18nSmokeCaseResult] 

32 by_language: dict[str, tuple[int, int]] # language -> (passed, total) 

33 

34 

35def run_i18n_smoke(engine: ClassificationEngine, path: str) -> I18nSmokeReport: 

36 raw = load_yaml_file(path) 

37 results: list[I18nSmokeCaseResult] = [] 

38 

39 for case in raw.get("cases", []): 

40 result = engine.classify(ClassifyRequest(text=case["text"], language=case["language"])) 

41 results.append( 

42 I18nSmokeCaseResult( 

43 language=case["language"], 

44 text=case["text"], 

45 expected_is_harmful=case["expected_is_harmful"], 

46 actual_is_harmful=result.is_harmful, 

47 ) 

48 ) 

49 

50 by_language: dict[str, list[int]] = {} 

51 for r in results: 

52 counts = by_language.setdefault(r.language, [0, 0]) 

53 counts[1] += 1 

54 if r.passed: 

55 counts[0] += 1 

56 

57 return I18nSmokeReport( 

58 total=len(results), 

59 passed=sum(1 for r in results if r.passed), 

60 failed_cases=[r for r in results if not r.passed], 

61 by_language={lang: (counts[0], counts[1]) for lang, counts in by_language.items()}, 

62 )