Integrate violation probability scoring

pyproject.toml

@@ -130,6 +130,7 @@ namespaces = true
 
 [tool.setuptools.package-data]
 "modelgauge.config_templates" = ["*.toml"]
+"modelbench.external_scoring" = ["*.json"]
 
 [tool.pytest.ini_options]
 addopts = [

src/modelbench/benchmark_runner.py

@@ -17,7 +17,7 @@
 from modelbench.benchmarks import BenchmarkDefinition, BenchmarkScore
 from modelbench.cache import DiskCache, MBCache
 from modelbench.run_journal import RunJournal
-from modelgauge.annotation import MissingAnnotationError
+from modelgauge.annotation import MissingAnnotationError, SafetyAnnotation
 from modelgauge.annotator import Annotator
 from modelgauge.annotator_registry import ANNOTATORS
 from modelgauge.base_test import PromptResponseTest, TestResult
@@ -260,9 +260,9 @@ def compile_annotations(self) -> list:
                             "hazard": hazard,
                             "prompt": item.test_item.prompt.text,
                             "response": item.sut_response.text,
-                            "is_safe": annotation.is_safe,
                             "is_valid": annotation.is_valid,
                         }
+                        ann.update(annotation.dump_value())
                         annotations.append(ann)
         return annotations
 
@@ -416,7 +416,7 @@ def fix_items_with_empty_responses(self, item):
         if item.sut_response.text is not None and item.sut_response.text.strip() == "":
             overridden_annotators: list[str] = []
             for annotator_uid, annotation in item.annotations.items():
-                if not annotation.is_safe:
+                if isinstance(annotation, SafetyAnnotation) and not annotation.is_safe:
                     annotation.is_safe = True
                     overridden_annotators.append(annotator_uid)
             # log if any annotations were overridden

src/modelbench/benchmark_runner_items.py

@@ -5,7 +5,7 @@
 from datetime import datetime, timezone
 from typing import List, Mapping
 
-from modelgauge.annotation import SafetyAnnotation
+from modelgauge.annotation import SafetyAnnotation, ViolationProbabilityAnnotation
 from modelgauge.annotator import Annotator
 from modelgauge.base_test import PromptResponseTest
 from modelgauge.dependency_helper import FromSourceDependencyHelper
@@ -89,7 +89,7 @@ class TestRunItem:
     test_item: TestItem
     sut: PromptResponseSUT = None
     sut_response: SUTResponse = None
-    annotations: dict[str, SafetyAnnotation] = dataclasses.field(default_factory=dict)
+    annotations: dict[str, SafetyAnnotation | ViolationProbabilityAnnotation] = dataclasses.field(default_factory=dict)
     measurements: dict[str, float] = dataclasses.field(default_factory=dict)
     failed = False
     exceptions: list = dataclasses.field(default_factory=list)

src/modelbench/benchmarks.py

@@ -8,14 +8,15 @@
 
 from modelbench.benchmark_runner_items import TestRunItem
 from modelbench.benchmark_score import BenchmarkScore, SecurityScore
+from modelbench.external_scoring import security_v_1_1_scoring
 from modelbench.external_scoring.scoring_adapters import RegressionScorer
 from modelbench.hazards import (
     HazardDefinition,
     HazardScore,
     SafeHazardV1_1,
     SafeHazardV1_2,
     SecurityJailbreakHazardV1_0_2,
-    SecurityJailbreakHazardV1_1,
+    SecurityHazardV1_1,
     SecurityNaiveHazardV1_0_2,
 )
 from modelbench.standards import NoStandardsFileError, NullStandards, Standards
@@ -277,12 +278,12 @@ class SecurityBenchmarkV1_1(GeneralPurposeAiChatBenchmarkV1):
     VERSION = "1.1"
     PROMPT_SETS = SECURITY_1_1_JAILBREAK_PROMPT_SETS  # pyright: ignore
 
-    def score(self, sut, hazard_scores, benchmark_end_time):
-        return SecurityScore(self, sut, hazard_scores, benchmark_end_time)
+    def score_benchmark_run(self, benchmark_run, sut):
+        return security_v_1_1_scoring.score_run(self, benchmark_run, sut)
 
     def _make_hazards(self) -> Sequence[HazardDefinition]:
         return [
-            SecurityJailbreakHazardV1_1(self.locale, self.prompt_set, self.evaluator),
+            SecurityHazardV1_1(self.locale, self.prompt_set, self.evaluator),
         ]
 
     _uid_definition = {

src/modelbench/consistency_checker.py

@@ -403,6 +403,33 @@ def failure_message(self) -> str:
         return f"{self.hazard}: SUT {self.sut}'s hazard score({self.journal_score}) is not a float in range [{self.MIN_SCORE}, {self.MAX_SCORE}]."
 
 
+class ViolationProbabilitiesInRange(JournalCheck):
+    """Security 1.1 records a violation probability per item and no hazard score."""
+
+    def __init__(self, search_engine: JournalSearch, sut, hazard):
+        self.sut = sut
+        self.hazard = hazard
+        self.invalid = []
+
+        hazard_info = search_engine.query("hazard info", hazard=self.hazard)
+        assert len(hazard_info) == 1, "Expected exactly 1 `hazard info` entry."
+        for test in hazard_info[0]["tests"]:
+            for entry in search_engine.query("measured item quality", sut=self.sut, test=test):
+                probability = entry["measurements"].get("violation_probability")
+                if not isinstance(probability, float) or not 0.0 <= probability <= 1.0:
+                    self.invalid.append((entry["prompt_id"], probability))
+
+    def check(self) -> bool:
+        return len(self.invalid) == 0
+
+    def failure_message(self) -> str:
+        assert not self.check()
+        return (
+            f"{self.hazard}: SUT {self.sut} has {len(self.invalid)} measured items whose "
+            f"violation_probability is not a float in [0, 1]."
+        )
+
+
 class JournalEntityLevelCheck:
     """A group of a checks that occupy the same conceptual entity-level in the journal.
 
@@ -491,12 +518,17 @@ def benchmark_has_multiple_annotators(benchmark: str) -> bool:
         # PRIVATE_EVALUATOR refers to the evaluator.
         return "general" in benchmark and "1.2" not in benchmark and benchmark.endswith(PRIVATE_EVALUATOR)
 
+    @staticmethod
+    def benchmark_measures_violation_probability(benchmark: str) -> bool:
+        # TODO: This would be easier if we can access the benchmark object instead of just relying on the uid.
+        return "security" in benchmark and "1.1" in benchmark
+
     @staticmethod
     def benchmark_score_is_frac_safe(benchmark: str) -> bool:
         # TODO: This would be easier if we can access the benchmark object instead of just relying on the uid.
         if "general" in benchmark and "1.2" in benchmark:
             return False
-        if "security" in benchmark and "1.1" in benchmark:
+        if ConsistencyChecker.benchmark_measures_violation_probability(benchmark):
             return False
         return True
 
@@ -570,7 +602,9 @@ def _init_checkers(self):
         )
         if self.hazards is not None:
             # Only run hazard checks if we are able to pull hazards from the journal.
-            if self.benchmark_score_is_frac_safe(self.benchmark):
+            if self.benchmark_measures_violation_probability(self.benchmark):
+                hazard_sut_checks = [ViolationProbabilitiesInRange]
+            elif self.benchmark_score_is_frac_safe(self.benchmark):
                 # TODO: Add checks for numeric grade and letter grade.
                 hazard_sut_checks = [HazardScoreIsFracSafe]
             else: