diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 636ef3c2..2a0f3a5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -336,7 +336,7 @@ jobs: - name: Verify specialist references without model downloads run: | pip install pytest safetensors pyte onnx==1.23.2 onnxruntime==1.31.0 onnxscript==0.7.2 'transformers>=4.51,<5' 'timestral @ git+https://github.com/mistralai/timestral.git@a534bdbd244d64f3d9d743f46c7bf6fc330ed4ff' - python -m pytest tests/test_specialist_labs.py tests/test_specialist_workflow.py tests/test_specialist_terminal_portability.py tests/test_specialist_native_export.py tests/test_specialist_forecast_portability.py tests/test_specialist_decision_portability.py -q + python -m pytest tests/test_specialist_labs.py tests/test_specialist_workflow.py tests/test_specialist_terminal_portability.py tests/test_specialist_native_export.py tests/test_specialist_native_scoring.py tests/test_specialist_forecast_portability.py tests/test_specialist_decision_portability.py -q - name: Upload coverage uses: actions/upload-artifact@v7.0.1 diff --git a/docs/factory/specialist-labs.md b/docs/factory/specialist-labs.md index f9a41a90..40ff35f4 100644 --- a/docs/factory/specialist-labs.md +++ b/docs/factory/specialist-labs.md @@ -335,6 +335,41 @@ not numerical or calibration parity. Refit temperature and rerun primary and regression gates before using native confidence. No model is promoted by this conversion. +For a source-bound native score run, re-export the adapter with training +identities retained, then use the existing local executable and model files: + +```sh +uv run --project scripts/specialists python scripts/specialists/decision.py score-native --binary --base --adapter --source-revision --data --seconds 60 --out +``` + +The scorer observes the executable, base/tokenizer, adapter and sidecar hashes, +and binds tokenizer versions and the exact prompt implementation into a distinct +native model identity. The source revision is operator-declared, not verified +build provenance. Every exact-token response must match its request. Changed +files, unknown adapter training provenance, malformed output and expired budgets +fail closed. Native adapter sidecars exported before training identities were +preserved require re-export from their original source adapter. +Merged bases retain their own training exclusions too. Standalone tokenizer +files and the resolved chat template are part of this identity. Checksums run at +the start and before final output; per-row file fingerprints detect intervening +changes. Every row cold-loads the native model in a separate bounded process. +The total time budget includes those loads; failure produces no final score +file. Use a small split and retain failed-run diagnostics separately. +All rows are checked against the native 256-token cap before any model process. +Metal libraries, bundles and dynamic libraries beside the executable or in its +packaged app's `Contents/Resources` and `Contents/Frameworks` are hashed too. +Missing Metal libraries block scoring. This does not attest external system +libraries or drivers. Exports with unknown +training provenance remain loadable diagnostic artifacts; re-exporting such a +source cannot reconstruct its training identities or qualify confidence. + +Use `decision.py calibrate` on native calibration scores only, with train, +primary and regression scores supplied through `--exclude`; score those splits +separately. Existing calibration and eval leakage checks remain authoritative. +MLX and browser calibration cannot qualify this different runtime identity. +The subprocess fixtures test the scoring boundary; actual native runs and a +frontier-qualified frozen primary/regression comparison remain separate proof. + ## Portable Qwen decisions Use a local tied-embedding Qwen3 MLX affine q4/group64 base and, optionally, diff --git a/scripts/specialists/decision.py b/scripts/specialists/decision.py index d92e2070..19fc8473 100644 --- a/scripts/specialists/decision.py +++ b/scripts/specialists/decision.py @@ -535,6 +535,14 @@ def argument_parser(): child.add_argument("--base", required=True) child.add_argument("--adapter", required=True) child.add_argument("--out", required=True) + child = sub.add_parser("score-native") + child.add_argument("--binary", required=True) + child.add_argument("--base", required=True) + child.add_argument("--source-revision", required=True) + child.add_argument("--adapter") + child.add_argument("--data", required=True) + child.add_argument("--out", required=True) + child.add_argument("--seconds", type=float, default=60) child = sub.add_parser("export-browser") child.add_argument("--base", required=True) child.add_argument("--adapter") @@ -572,12 +580,32 @@ def run_model(args): print(json.dumps(budget.measurements())) -def main(): - args = argument_parser().parse_args() +def run_native(args): if args.command == "export-native": from decision_native import export print(json.dumps(export(args.base, args.adapter, args.out))) + return + from decision_native_score import NativeDecisionModel + + budget = Budget(args.seconds) + rows = read_rows(args.data) + if Path(args.out).exists(): + raise FileExistsError(args.out) + model = NativeDecisionModel( + args.binary, args.base, args.source_revision, args.adapter, budget + ) + model.preflight(rows) + scores = [model.score(row) for row in rows] + model.finish() + write_rows(args.out, scores) + print(json.dumps(budget.measurements())) + + +def main(): + args = argument_parser().parse_args() + if args.command in ("score-native", "export-native"): + run_native(args) elif args.command == "export-browser": from decision_browser import export diff --git a/scripts/specialists/decision_native.py b/scripts/specialists/decision_native.py index 5bd7c75a..bf0dda60 100644 --- a/scripts/specialists/decision_native.py +++ b/scripts/specialists/decision_native.py @@ -136,6 +136,7 @@ def export(base, adapter, out): "base_identity": metadata["base_identity"], "source_adapter_sha256": metadata["adapter_sha256"], "native_adapter_sha256": file_hash(out), + "train_identities": metadata.get("train_identities"), "entries": len(entries), "quality": "unqualified; recalibrate after runtime conversion", } diff --git a/scripts/specialists/decision_native_score.py b/scripts/specialists/decision_native_score.py new file mode 100644 index 00000000..13614c12 --- /dev/null +++ b/scripts/specialists/decision_native_score.py @@ -0,0 +1,295 @@ +"""Score exact native option logits with observed artifact and prompt identities. + +Contract: docs/factory/specialist-labs.md. The source revision is declared by +the operator; a binary checksum does not prove which source built that binary. +""" + +from __future__ import annotations + +import json +import importlib.metadata +import math +import os +import re +import subprocess +import tempfile +import time +from pathlib import Path + +from common import Budget, digest, file_hash, read_json +from decision import identity, option_ids, prompt, validate_item +from decision_native import base_files + + +class NativeDecisionModel: + def __init__( + self, binary, base, source_revision, adapter=None, budget=None, tokenizer=None + ): + self.binary, self.base = Path(binary).resolve(), Path(base).resolve() + self.adapter = Path(adapter).resolve() if adapter else None + self.budget = budget or Budget() + if not re.fullmatch(r"[0-9a-f]{40}", source_revision): + raise ValueError("native source revision must be a declared full commit") + if not self.binary.is_file() or not os.access(self.binary, os.X_OK): + raise ValueError("native binary must be an existing executable file") + self.training_identities = set() + self.artifacts = self.snapshot() + self.fingerprints = self.file_fingerprints() + if (self.base / "specialist.json").exists(): + self.read_training_identities(read_json(self.base / "specialist.json")) + self.check_adapter() + if tokenizer is None: + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained( + str(self.base), local_files_only=True, trust_remote_code=False + ) + self.tokenizer = tokenizer + tokenizer_versions = {} + for package in ("transformers", "tokenizers", "jinja2", "markupsafe"): + try: + tokenizer_versions[package] = importlib.metadata.version(package) + except importlib.metadata.PackageNotFoundError: + tokenizer_versions[package] = None + self.runtime_identity = { + "backend": "native-exact-token-logits-v1", + "source_revision_declared": source_revision, + "artifacts_observed": self.artifacts, + "scorer_sha256": file_hash(__file__), + "prompt_implementation_sha256": file_hash( + Path(__file__).with_name("decision.py") + ), + "tokenizer_runtime": tokenizer_versions, + "tokenizer_class": type(tokenizer).__module__ + + "." + + type(tokenizer).__qualname__, + "resolved_chat_template_sha256": digest( + getattr(tokenizer, "chat_template", None) + ), + } + self.model_identity = digest(self.runtime_identity) + + def snapshot(self): + files = base_files(self.base) + if "config.json" not in files or not any( + name.endswith(".safetensors") for name in files + ): + raise ValueError("native base requires config and local model weights") + resources = self.native_resources() + if not any(path.suffix == ".metallib" for path in resources.values()): + raise ValueError( + "native Metal kernel resources unavailable; retain the CLI's Metal library" + ) + result = { + "binary_sha256": file_hash(self.binary), + "base_files": files, + "tokenizer_files": { + str(path.relative_to(self.base)): file_hash(path) + for path in self.tokenizer_files() + }, + "native_resources": { + name: file_hash(path) for name, path in resources.items() + }, + } + if self.adapter: + result["adapter_sha256"] = file_hash(self.adapter) + result["adapter_provenance_sha256"] = file_hash(str(self.adapter) + ".json") + return result + + def native_resources(self): + roots = {"binary": self.binary.parent} + if ( + self.binary.parent.name == "MacOS" + and self.binary.parent.parent.name == "Contents" + ): + roots["app-resources"] = self.binary.parent.parent / "Resources" + roots["app-frameworks"] = self.binary.parent.parent / "Frameworks" + resources = {} + for name, root in roots.items(): + paths = set(root.glob("*.metallib")) | set(root.glob("*.dylib")) + for bundle in list(root.glob("*.bundle")) + list(root.glob("*.framework")): + paths.update(path for path in bundle.rglob("*") if path.is_file()) + resources.update( + { + name + "/" + str(path.relative_to(root)): path + for path in sorted(paths) + if path.is_file() + } + ) + return resources + + def tokenizer_files(self): + return sorted( + { + path + for path in self.base.iterdir() + if path.is_file() + and path.suffix in (".jinja", ".txt", ".model", ".tiktoken") + } + | set((self.base / "chat_templates").glob("*.jinja")) + ) + + def file_fingerprints(self): + paths = {"binary": self.binary} + paths.update( + { + "base/" + path.name: path + for path in self.base.iterdir() + if path.is_file() and path.suffix in (".json", ".safetensors") + } + ) + paths.update( + {"native/" + name: path for name, path in self.native_resources().items()} + ) + paths.update( + { + "base/" + str(path.relative_to(self.base)): path + for path in self.tokenizer_files() + } + ) + if self.adapter: + paths.update( + { + "adapter": self.adapter, + "adapter_provenance": Path(str(self.adapter) + ".json"), + } + ) + result = {} + for name, path in paths.items(): + status = path.stat() + result[name] = ( + status.st_dev, + status.st_ino, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + return result + + def check_adapter(self): + if not self.adapter: + return + metadata = read_json(str(self.adapter) + ".json") + if ( + metadata.get("schema") != "decision-native-adapter-v1" + or metadata.get("base_files") != self.artifacts["base_files"] + or metadata.get("native_adapter_sha256") != self.artifacts["adapter_sha256"] + ): + raise ValueError("native adapter/base provenance mismatch") + self.read_training_identities(metadata) + + def read_training_identities(self, metadata): + identities = metadata.get("train_identities") + if ( + not isinstance(identities, list) + or not identities + or any( + not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value) + for value in identities + ) + or len(set(identities)) != len(identities) + ): + raise ValueError( + "native training provenance missing or invalid; source must record original training identities" + ) + self.training_identities.update(identities) + + def unchanged(self): + self.budget.check() + if self.file_fingerprints() != self.fingerprints: + raise ValueError("native scoring artifacts changed; discard this score run") + self.budget.check() + + def finish(self): + self.unchanged() + if self.snapshot() != self.artifacts: + raise ValueError("native scoring artifacts changed; discard this score run") + self.budget.check() + + def prepare(self, item): + labels = validate_item(item) + ids = option_ids(labels, self.tokenizer) + tokens = self.tokenizer.encode( + prompt(item, self.tokenizer), add_special_tokens=False + ) + maximum = min( + 256, read_json(self.base / "config.json")["max_position_embeddings"] + ) + if ( + not tokens + or len(tokens) > maximum + or any(type(token) is not int or token < 0 for token in tokens + ids) + ): + raise ValueError( + "native prompt requires bounded exact token ids; no truncation" + ) + return labels, ids, tokens + + def preflight(self, rows): + self.unchanged() + for item in rows: + self.prepare(item) + self.budget.check() + + def score(self, item): + self.unchanged() + labels, ids, tokens = self.prepare(item) + started = time.monotonic() + with tempfile.TemporaryDirectory(prefix="posttrain-native-probe-") as directory: + probe = Path(directory) / "probe.json" + probe.write_text(json.dumps({"tokens": tokens, "labels": ids})) + command = [ + str(self.binary), + "debug-logits", + "--hf", + str(self.base), + "--probe", + str(probe), + ] + if self.adapter: + command += ["--lora", str(self.adapter)] + remaining = self.budget.seconds - (time.monotonic() - self.budget.started) + if remaining <= 0: + raise TimeoutError("specialist work budget exhausted") + try: + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=remaining, + check=True, + ) + except subprocess.CalledProcessError as error: + raise ValueError( + f"native exact-token probe failed ({error.returncode}): {error.stderr.strip()[-1000:]}" + ) from None + self.unchanged() + value = json.loads(result.stdout) + if not isinstance(value, dict): + raise ValueError("native exact-token response does not match the request") + logits = value.get("logits") + if ( + value.get("schema") != "native-exact-token-logits-v1" + or value.get("tokens") != len(tokens) + or value.get("labels") != ids + or not isinstance(logits, list) + or len(logits) != len(labels) + or any( + type(number) not in (int, float) or not math.isfinite(number) + for number in logits + ) + ): + raise ValueError("native exact-token response does not match the request") + return { + "id": item.get("id", identity(item)), + "identity": identity(item), + "labels": labels, + "logits": logits, + "model_identity": self.model_identity, + "runtime_identity": self.runtime_identity, + "answer": item.get("answer"), + "prompt_tokens": len(tokens), + "probe_sha256": digest({"tokens": tokens, "labels": ids}), + "seen_in_training": identity(item) in self.training_identities, + "elapsed_seconds": time.monotonic() - started, + } diff --git a/tests/test_specialist_native_export.py b/tests/test_specialist_native_export.py index b9003a39..5d34f769 100644 --- a/tests/test_specialist_native_export.py +++ b/tests/test_specialist_native_export.py @@ -56,6 +56,7 @@ def fixture(tmp_path): "base_identity": digest({"backend": "mlx", "files": base_files(base)}), "adapter_sha256": file_hash(adapter / "adapters.safetensors"), "adapter_config_sha256": file_hash(adapter / "adapter_config.json"), + "train_identities": [digest("training-example")], } (adapter / "specialist.json").write_text(json.dumps(metadata)) return base, adapter @@ -80,6 +81,7 @@ def test_native_export_preserves_subset_scaling_and_bytes(tmp_path): ) assert receipt["native_adapter_sha256"] == file_hash(out) assert receipt["base_files"] == base_files(base) + assert receipt["train_identities"] == [digest("training-example")] with pytest.raises(FileExistsError): export(base, adapter, out) @@ -97,6 +99,33 @@ def test_native_export_rejects_changed_base_or_adapter(tmp_path): export(base, adapter, tmp_path / "bad.lora") +def test_native_export_operator_command_preserves_identity( + tmp_path, monkeypatch, capsys +): + import decision + + base, adapter = fixture(tmp_path) + out = tmp_path / "operator.lora" + monkeypatch.setattr( + sys, + "argv", + [ + "decision.py", + "export-native", + "--base", + str(base), + "--adapter", + str(adapter), + "--out", + str(out), + ], + ) + decision.main() + receipt = json.loads(capsys.readouterr().out) + assert receipt["native_adapter_sha256"] == file_hash(out) + assert receipt["train_identities"] == [digest("training-example")] + + def test_native_float32_conversion_rejects_overflow(tmp_path): from safetensors.numpy import load_file diff --git a/tests/test_specialist_native_scoring.py b/tests/test_specialist_native_scoring.py new file mode 100644 index 00000000..34de4166 --- /dev/null +++ b/tests/test_specialist_native_scoring.py @@ -0,0 +1,350 @@ +"""Native scores bind real subprocess requests and reject stale calibration.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts" / "specialists")) +from common import Budget, digest, file_hash +from decision import calibrate, identity, metrics, prediction +from decision_native import base_files +from decision_native_score import NativeDecisionModel + + +class Tokenizer: + def apply_chat_template(self, messages, **kwargs): + return messages[0]["content"] + + def encode(self, text, **kwargs): + return {"A": [32], "B": [33]}.get(text, [1, 2, 3]) + + +def item(name="test"): + return { + "id": name, + "question": name, + "options": {"A": "yes", "B": "no"}, + "answer": "A", + } + + +def fixture(tmp_path, change=""): + base = tmp_path / "base" + base.mkdir() + (base / "config.json").write_text(json.dumps({"max_position_embeddings": 256})) + (base / "model.safetensors").write_bytes(b"model-fixture") + (base / "tokenizer.json").write_text("{}") + binary = tmp_path / "posttrainllm" + binary.write_text( + f"#!{Path(sys.executable).resolve()}\n" + "import json, sys\nfrom pathlib import Path\n" + "probe = json.loads(Path(sys.argv[sys.argv.index('--probe') + 1]).read_text())\n" + "value = {'schema': 'native-exact-token-logits-v1', 'tokens': len(probe['tokens']), 'labels': probe['labels'], 'logits': [3.0, 1.0]}\n" + + change + + "\nprint(json.dumps(value))\n" + ) + binary.chmod(0o700) + (binary.parent / "default.metallib").write_bytes(b"process-fixture-only-metal") + return base, binary + + +def model(base, binary, adapter=None, tokenizer=None): + return NativeDecisionModel( + binary, base, "a" * 40, adapter, tokenizer=tokenizer or Tokenizer() + ) + + +def test_native_subprocess_score_and_backend_specific_calibration(tmp_path): + base, binary = fixture(tmp_path) + native = model(base, binary) + scores = [native.score(item(f"calibration-{i}")) for i in range(4)] + primary = [native.score(item("primary"))] + regression = [native.score(item("regression"))] + fitted = calibrate(scores, primary + regression) + assert metrics(primary, fitted)["accuracy"] == 1 + assert metrics(regression, fitted)["accuracy"] == 1 + assert scores[0]["logits"] == [3.0, 1.0] + assert scores[0]["runtime_identity"]["source_revision_declared"] == "a" * 40 + assert scores[0]["runtime_identity"]["artifacts_observed"][ + "binary_sha256" + ] == file_hash(binary) + assert scores[0]["probe_sha256"] == digest( + {"tokens": [1, 2, 3], "labels": [32, 33]} + ) + with pytest.raises(ValueError, match="overlap"): + metrics(scores, fitted) + with pytest.raises(ValueError, match="another model"): + prediction({**primary[0], "model_identity": "mlx-source"}, fitted) + changed_revision = NativeDecisionModel( + binary, base, "b" * 40, tokenizer=Tokenizer() + ) + with pytest.raises(ValueError, match="another model"): + prediction(changed_revision.score(item()), fitted) + + +@pytest.mark.parametrize("filename", ["model.safetensors", "tokenizer.json"]) +def test_native_scoring_rejects_changed_artifacts(tmp_path, filename): + base, binary = fixture(tmp_path) + native = model(base, binary) + (base / filename).write_bytes(b"changed") + with pytest.raises(ValueError, match="artifacts changed"): + native.score(item()) + + +def test_native_scoring_rejects_change_during_the_actual_process(tmp_path): + change = "Path(sys.argv[sys.argv.index('--hf') + 1], 'model.safetensors').write_bytes(b'changed')\n" + base, binary = fixture(tmp_path, change) + with pytest.raises(ValueError, match="artifacts changed"): + model(base, binary).score(item()) + + +def test_native_timeout_reaps_the_owned_process(tmp_path): + pid_file = tmp_path / "native.pid" + change = ( + f"import os, time\nPath({str(pid_file)!r}).write_text(str(os.getpid()))\n" + "time.sleep(30)\n" + ) + base, binary = fixture(tmp_path, change) + native = NativeDecisionModel( + binary, base, "a" * 40, budget=Budget(3), tokenizer=Tokenizer() + ) + with pytest.raises(subprocess.TimeoutExpired): + native.score(item()) + with pytest.raises(ProcessLookupError): + os.kill(int(pid_file.read_text()), 0) + + +@pytest.mark.parametrize( + "change", + [ + "value['labels'] = [33, 32]", + "value['tokens'] = 4", + "value['logits'] = [1.0]", + "value['logits'] = [float('nan'), 1.0]", + "value['logits'] = [True, 1.0]", + ], +) +def test_native_scoring_rejects_mismatched_response(tmp_path, change): + base, binary = fixture(tmp_path, change) + with pytest.raises(ValueError, match="response does not match"): + model(base, binary).score(item()) + + +def test_native_adapter_training_provenance_and_calibration_leakage(tmp_path): + base, binary = fixture(tmp_path) + adapter = tmp_path / "native.lora" + adapter.write_bytes(b"adapter-fixture") + sidecar = Path(str(adapter) + ".json") + metadata = { + "schema": "decision-native-adapter-v1", + "base_files": base_files(base), + "native_adapter_sha256": file_hash(adapter), + } + sidecar.write_text(json.dumps(metadata)) + with pytest.raises(ValueError, match="training provenance"): + model(base, binary, adapter) + metadata["train_identities"] = [identity(item("train"))] + sidecar.write_text(json.dumps(metadata)) + native = model(base, binary, adapter) + trained = native.score(item("train")) + assert trained["seen_in_training"] is True + with pytest.raises(ValueError, match="training examples"): + calibrate([trained]) + assert native.model_identity != model(base, binary).model_identity + adapter.write_bytes(b"changed-adapter") + with pytest.raises(ValueError, match="provenance mismatch"): + model(base, binary, adapter) + + +def test_native_merged_base_preserves_training_exclusion(tmp_path): + base, binary = fixture(tmp_path) + (base / "specialist.json").write_text( + json.dumps({"train_identities": [identity(item("train"))]}) + ) + native = model(base, binary) + with pytest.raises(ValueError, match="training examples"): + calibrate([native.score(item("train"))]) + + +def test_native_identity_binds_standalone_chat_template(tmp_path): + base, binary = fixture(tmp_path) + old = model(base, binary) + (base / "chat_template.jinja").write_text("changed prompt template") + with pytest.raises(ValueError, match="artifacts changed"): + old.score(item()) + assert model(base, binary).model_identity != old.model_identity + + +def test_native_identity_binds_template_renderer_version(tmp_path, monkeypatch): + import decision_native_score + + base, binary = fixture(tmp_path) + native = model(base, binary) + version = decision_native_score.importlib.metadata.version + monkeypatch.setattr( + decision_native_score.importlib.metadata, + "version", + lambda name: "changed-renderer" if name == "jinja2" else version(name), + ) + assert model(base, binary).model_identity != native.model_identity + + +def test_native_identity_binds_colocated_metal_resources(tmp_path): + base, binary = fixture(tmp_path) + bundle = binary.parent / "mlx-swift_Cmlx.bundle" + bundle.mkdir() + shader = bundle / "default.metallib" + shader.write_bytes(b"compiled-metal-fixture") + native = model(base, binary) + shader.write_bytes(b"changed-metal-fixture") + with pytest.raises(ValueError, match="artifacts changed"): + native.score(item()) + assert model(base, binary).model_identity != native.model_identity + + +def test_native_identity_binds_packaged_app_resources(tmp_path): + base, binary = fixture(tmp_path) + contents = tmp_path / "posttrainllm.app/Contents" + destination = contents / "MacOS/posttrainllm-cli" + destination.parent.mkdir(parents=True) + binary.rename(destination) + bundle = contents / "Resources/mlx-swift_Cmlx.bundle" + bundle.mkdir(parents=True) + shader = bundle / "default.metallib" + shader.write_bytes(b"packaged-metal-fixture") + native = model(base, destination) + shader.write_bytes(b"changed-packaged-metal-fixture") + with pytest.raises(ValueError, match="artifacts changed"): + native.score(item()) + assert model(base, destination).model_identity != native.model_identity + + +def test_native_missing_metal_resources_cannot_qualify_scores(tmp_path): + base, binary = fixture(tmp_path) + (binary.parent / "default.metallib").unlink() + with pytest.raises(ValueError, match="Metal kernel resources unavailable"): + model(base, binary) + + +def test_native_preflight_rejects_late_oversize_before_any_process( + tmp_path, monkeypatch +): + import decision + import decision_native_score + + marker = tmp_path / "native-called" + base, binary = fixture(tmp_path, f"Path({str(marker)!r}).write_text('called')") + + class LongTokenizer(Tokenizer): + def encode(self, text, **kwargs): + if "too-long" in text: + return [1] * 257 + return super().encode(text, **kwargs) + + native = model(base, binary, tokenizer=LongTokenizer()) + data, out = tmp_path / "data.jsonl", tmp_path / "scores.jsonl" + data.write_text( + "\n".join(json.dumps(row) for row in [item("valid"), item("too-long")]) + "\n" + ) + monkeypatch.setattr( + decision_native_score, "NativeDecisionModel", lambda *args: native + ) + monkeypatch.setattr( + sys, + "argv", + [ + "decision.py", + "score-native", + "--binary", + str(binary), + "--base", + str(base), + "--source-revision", + "a" * 40, + "--data", + str(data), + "--out", + str(out), + ], + ) + with pytest.raises(ValueError, match="bounded exact token"): + decision.main() + assert not marker.exists() + assert not out.exists() + + +def test_native_non_object_response_and_stderr_are_reported(tmp_path): + base, binary = fixture(tmp_path, "value = []") + with pytest.raises(ValueError, match="response does not match"): + model(base, binary).score(item()) + binary.write_text( + f"#!{Path(sys.executable).resolve()}\nimport sys\nsys.stderr.write('base identity mismatch')\nsys.exit(2)\n" + ) + with pytest.raises(ValueError, match="base identity mismatch"): + model(base, binary).score(item()) + + +def test_native_full_checksum_is_rechecked_at_finish(tmp_path, monkeypatch): + base, binary = fixture(tmp_path) + native = model(base, binary) + native.score(item()) + changed = {**native.artifacts, "binary_sha256": "changed"} + monkeypatch.setattr(native, "snapshot", lambda: changed) + with pytest.raises(ValueError, match="artifacts changed"): + native.finish() + + +def test_native_cli_does_not_publish_scores_if_final_validation_fails( + tmp_path, monkeypatch +): + import decision + import decision_native_score + + base, binary = fixture(tmp_path) + native = model(base, binary) + data, out = tmp_path / "data.jsonl", tmp_path / "scores.jsonl" + data.write_text(json.dumps(item()) + "\n") + changed = {**native.artifacts, "binary_sha256": "changed"} + monkeypatch.setattr(native, "snapshot", lambda: changed) + monkeypatch.setattr( + decision_native_score, "NativeDecisionModel", lambda *args: native + ) + monkeypatch.setattr( + sys, + "argv", + [ + "decision.py", + "score-native", + "--binary", + str(binary), + "--base", + str(base), + "--source-revision", + "a" * 40, + "--data", + str(data), + "--out", + str(out), + ], + ) + with pytest.raises(ValueError, match="artifacts changed"): + decision.main() + assert not out.exists() + + +@pytest.mark.parametrize( + "encoding, message", [([1, 2], "exactly one token"), ([32], "collide")] +) +def test_native_option_token_contract(tmp_path, encoding, message): + base, binary = fixture(tmp_path) + + class InvalidTokenizer(Tokenizer): + def encode(self, text, **kwargs): + return encoding + + with pytest.raises(ValueError, match=message): + model(base, binary, tokenizer=InvalidTokenizer()).score(item())