From b8178fc124ee8532e332b45f4c26cd9617631073 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Thu, 29 Oct 2026 15:23:08 +0530 Subject: [PATCH 1/6] Bind native decision scores to runtime artifacts and training provenance --- .github/workflows/ci.yml | 2 +- docs/factory/specialist-labs.md | 22 +++ scripts/specialists/decision.py | 22 ++- scripts/specialists/decision_native.py | 1 + scripts/specialists/decision_native_score.py | 174 +++++++++++++++++++ tests/test_specialist_native_export.py | 2 + tests/test_specialist_native_scoring.py | 173 ++++++++++++++++++ 7 files changed, 394 insertions(+), 2 deletions(-) create mode 100644 scripts/specialists/decision_native_score.py create mode 100644 tests/test_specialist_native_scoring.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 636ef3c2..2a0f3a5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -336,7 +336,7 @@ jobs: - name: Verify specialist references without model downloads run: | pip install pytest safetensors pyte onnx==1.23.2 onnxruntime==1.31.0 onnxscript==0.7.2 'transformers>=4.51,<5' 'timestral @ git+https://github.com/mistralai/timestral.git@a534bdbd244d64f3d9d743f46c7bf6fc330ed4ff' - python -m pytest tests/test_specialist_labs.py tests/test_specialist_workflow.py tests/test_specialist_terminal_portability.py tests/test_specialist_native_export.py tests/test_specialist_forecast_portability.py tests/test_specialist_decision_portability.py -q + python -m pytest tests/test_specialist_labs.py tests/test_specialist_workflow.py tests/test_specialist_terminal_portability.py tests/test_specialist_native_export.py tests/test_specialist_native_scoring.py tests/test_specialist_forecast_portability.py tests/test_specialist_decision_portability.py -q - name: Upload coverage uses: actions/upload-artifact@v7.0.1 diff --git a/docs/factory/specialist-labs.md b/docs/factory/specialist-labs.md index f9a41a90..91ff0981 100644 --- a/docs/factory/specialist-labs.md +++ b/docs/factory/specialist-labs.md @@ -335,6 +335,28 @@ not numerical or calibration parity. Refit temperature and rerun primary and regression gates before using native confidence. No model is promoted by this conversion. +For a source-bound native score run, re-export the adapter with training +identities retained, then use the existing local executable and model files: + +```sh +uv run --project scripts/specialists python scripts/specialists/decision.py score-native --binary --base --adapter --source-revision --data --seconds 60 --out +``` + +The scorer observes the executable, base/tokenizer, adapter and sidecar hashes, +and binds tokenizer versions and the exact prompt implementation into a distinct +native model identity. The source revision is operator-declared, not verified +build provenance. Every exact-token response must match its request. Changed +files, unknown adapter training provenance, malformed output and expired budgets +fail closed. Native adapter sidecars exported before training identities were +preserved require re-export from their original source adapter. + +Use `decision.py calibrate` on native calibration scores only, with train, +primary and regression scores supplied through `--exclude`; score those splits +separately. Existing calibration and eval leakage checks remain authoritative. +MLX and browser calibration cannot qualify this different runtime identity. +The subprocess fixtures test the scoring boundary; actual native runs and a +frontier-qualified frozen primary/regression comparison remain separate proof. + ## Portable Qwen decisions Use a local tied-embedding Qwen3 MLX affine q4/group64 base and, optionally, diff --git a/scripts/specialists/decision.py b/scripts/specialists/decision.py index d92e2070..4ad11754 100644 --- a/scripts/specialists/decision.py +++ b/scripts/specialists/decision.py @@ -535,6 +535,14 @@ def argument_parser(): child.add_argument("--base", required=True) child.add_argument("--adapter", required=True) child.add_argument("--out", required=True) + child = sub.add_parser("score-native") + child.add_argument("--binary", required=True) + child.add_argument("--base", required=True) + child.add_argument("--source-revision", required=True) + child.add_argument("--adapter") + child.add_argument("--data", required=True) + child.add_argument("--out", required=True) + child.add_argument("--seconds", type=float, default=60) child = sub.add_parser("export-browser") child.add_argument("--base", required=True) child.add_argument("--adapter") @@ -574,7 +582,19 @@ def run_model(args): def main(): args = argument_parser().parse_args() - if args.command == "export-native": + if args.command == "score-native": + from decision_native_score import NativeDecisionModel + + budget = Budget(args.seconds) + rows = read_rows(args.data) + if Path(args.out).exists(): + raise FileExistsError(args.out) + model = NativeDecisionModel( + args.binary, args.base, args.source_revision, args.adapter, budget + ) + write_rows(args.out, [model.score(row) for row in rows]) + print(json.dumps(budget.measurements())) + elif args.command == "export-native": from decision_native import export print(json.dumps(export(args.base, args.adapter, args.out))) diff --git a/scripts/specialists/decision_native.py b/scripts/specialists/decision_native.py index 5bd7c75a..bf0dda60 100644 --- a/scripts/specialists/decision_native.py +++ b/scripts/specialists/decision_native.py @@ -136,6 +136,7 @@ def export(base, adapter, out): "base_identity": metadata["base_identity"], "source_adapter_sha256": metadata["adapter_sha256"], "native_adapter_sha256": file_hash(out), + "train_identities": metadata.get("train_identities"), "entries": len(entries), "quality": "unqualified; recalibrate after runtime conversion", } diff --git a/scripts/specialists/decision_native_score.py b/scripts/specialists/decision_native_score.py new file mode 100644 index 00000000..8db61fb5 --- /dev/null +++ b/scripts/specialists/decision_native_score.py @@ -0,0 +1,174 @@ +"""Score exact native option logits with observed artifact and prompt identities. + +Contract: docs/factory/specialist-labs.md. The source revision is declared by +the operator; a binary checksum does not prove which source built that binary. +""" + +from __future__ import annotations + +import json +import importlib.metadata +import math +import os +import re +import subprocess +import tempfile +import time +from pathlib import Path + +from common import Budget, digest, file_hash, read_json +from decision import identity, option_ids, prompt, validate_item +from decision_native import base_files + + +class NativeDecisionModel: + def __init__( + self, binary, base, source_revision, adapter=None, budget=None, tokenizer=None + ): + self.binary, self.base = Path(binary).resolve(), Path(base).resolve() + self.adapter = Path(adapter).resolve() if adapter else None + self.budget = budget or Budget() + if not re.fullmatch(r"[0-9a-f]{40}", source_revision): + raise ValueError("native source revision must be a declared full commit") + if not self.binary.is_file() or not os.access(self.binary, os.X_OK): + raise ValueError("native binary must be an existing executable file") + self.training_identities = set() + self.artifacts = self.snapshot() + self.check_adapter() + if tokenizer is None: + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained( + str(self.base), local_files_only=True, trust_remote_code=False + ) + self.tokenizer = tokenizer + tokenizer_versions = {} + for package in ("transformers", "tokenizers"): + try: + tokenizer_versions[package] = importlib.metadata.version(package) + except importlib.metadata.PackageNotFoundError: + tokenizer_versions[package] = None + self.runtime_identity = { + "backend": "native-exact-token-logits-v1", + "source_revision_declared": source_revision, + "artifacts_observed": self.artifacts, + "scorer_sha256": file_hash(__file__), + "prompt_implementation_sha256": file_hash( + Path(__file__).with_name("decision.py") + ), + "tokenizer_runtime": tokenizer_versions, + "tokenizer_class": type(tokenizer).__module__ + + "." + + type(tokenizer).__qualname__, + } + self.model_identity = digest(self.runtime_identity) + + def snapshot(self): + files = base_files(self.base) + if "config.json" not in files or not any( + name.endswith(".safetensors") for name in files + ): + raise ValueError("native base requires config and local model weights") + result = {"binary_sha256": file_hash(self.binary), "base_files": files} + if self.adapter: + result["adapter_sha256"] = file_hash(self.adapter) + result["adapter_provenance_sha256"] = file_hash(str(self.adapter) + ".json") + return result + + def check_adapter(self): + if not self.adapter: + return + metadata = read_json(str(self.adapter) + ".json") + identities = metadata.get("train_identities") + if ( + metadata.get("schema") != "decision-native-adapter-v1" + or metadata.get("base_files") != self.artifacts["base_files"] + or metadata.get("native_adapter_sha256") != self.artifacts["adapter_sha256"] + ): + raise ValueError("native adapter/base provenance mismatch") + if ( + not isinstance(identities, list) + or not identities + or any( + not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value) + for value in identities + ) + or len(set(identities)) != len(identities) + ): + raise ValueError( + "native adapter training provenance missing or invalid; re-export" + ) + self.training_identities = set(identities) + + def unchanged(self): + self.budget.check() + if self.snapshot() != self.artifacts: + raise ValueError("native scoring artifacts changed; discard this score run") + self.budget.check() + + def score(self, item): + self.unchanged() + labels = validate_item(item) + ids = option_ids(labels, self.tokenizer) + tokens = self.tokenizer.encode( + prompt(item, self.tokenizer), add_special_tokens=False + ) + maximum = min( + 256, read_json(self.base / "config.json")["max_position_embeddings"] + ) + if ( + not tokens + or len(tokens) > maximum + or any(type(token) is not int or token < 0 for token in tokens + ids) + ): + raise ValueError( + "native prompt requires bounded exact token ids; no truncation" + ) + started = time.monotonic() + with tempfile.TemporaryDirectory(prefix="posttrain-native-probe-") as directory: + probe = Path(directory) / "probe.json" + probe.write_text(json.dumps({"tokens": tokens, "labels": ids})) + command = [ + str(self.binary), + "debug-logits", + "--hf", + str(self.base), + "--probe", + str(probe), + ] + if self.adapter: + command += ["--lora", str(self.adapter)] + remaining = self.budget.seconds - (time.monotonic() - self.budget.started) + if remaining <= 0: + raise TimeoutError("specialist work budget exhausted") + result = subprocess.run( + command, capture_output=True, text=True, timeout=remaining, check=True + ) + self.unchanged() + value = json.loads(result.stdout) + logits = value.get("logits") + if ( + value.get("schema") != "native-exact-token-logits-v1" + or value.get("tokens") != len(tokens) + or value.get("labels") != ids + or not isinstance(logits, list) + or len(logits) != len(labels) + or any( + type(number) not in (int, float) or not math.isfinite(number) + for number in logits + ) + ): + raise ValueError("native exact-token response does not match the request") + return { + "id": item.get("id", identity(item)), + "identity": identity(item), + "labels": labels, + "logits": logits, + "model_identity": self.model_identity, + "runtime_identity": self.runtime_identity, + "answer": item.get("answer"), + "prompt_tokens": len(tokens), + "probe_sha256": digest({"tokens": tokens, "labels": ids}), + "seen_in_training": identity(item) in self.training_identities, + "elapsed_seconds": time.monotonic() - started, + } diff --git a/tests/test_specialist_native_export.py b/tests/test_specialist_native_export.py index b9003a39..aba1e010 100644 --- a/tests/test_specialist_native_export.py +++ b/tests/test_specialist_native_export.py @@ -56,6 +56,7 @@ def fixture(tmp_path): "base_identity": digest({"backend": "mlx", "files": base_files(base)}), "adapter_sha256": file_hash(adapter / "adapters.safetensors"), "adapter_config_sha256": file_hash(adapter / "adapter_config.json"), + "train_identities": [digest("training-example")], } (adapter / "specialist.json").write_text(json.dumps(metadata)) return base, adapter @@ -80,6 +81,7 @@ def test_native_export_preserves_subset_scaling_and_bytes(tmp_path): ) assert receipt["native_adapter_sha256"] == file_hash(out) assert receipt["base_files"] == base_files(base) + assert receipt["train_identities"] == [digest("training-example")] with pytest.raises(FileExistsError): export(base, adapter, out) diff --git a/tests/test_specialist_native_scoring.py b/tests/test_specialist_native_scoring.py new file mode 100644 index 00000000..bfc64a1c --- /dev/null +++ b/tests/test_specialist_native_scoring.py @@ -0,0 +1,173 @@ +"""Native scores bind real subprocess requests and reject stale calibration.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts" / "specialists")) +from common import Budget, digest, file_hash +from decision import calibrate, identity, metrics, prediction +from decision_native import base_files +from decision_native_score import NativeDecisionModel + + +class Tokenizer: + def apply_chat_template(self, messages, **kwargs): + return messages[0]["content"] + + def encode(self, text, **kwargs): + return {"A": [32], "B": [33]}.get(text, [1, 2, 3]) + + +def item(name="test"): + return { + "id": name, + "question": name, + "options": {"A": "yes", "B": "no"}, + "answer": "A", + } + + +def fixture(tmp_path, change=""): + base = tmp_path / "base" + base.mkdir() + (base / "config.json").write_text(json.dumps({"max_position_embeddings": 256})) + (base / "model.safetensors").write_bytes(b"model-fixture") + (base / "tokenizer.json").write_text("{}") + binary = tmp_path / "posttrainllm" + binary.write_text( + f"#!{Path(sys.executable).resolve()}\n" + "import json, sys\nfrom pathlib import Path\n" + "probe = json.loads(Path(sys.argv[sys.argv.index('--probe') + 1]).read_text())\n" + "value = {'schema': 'native-exact-token-logits-v1', 'tokens': len(probe['tokens']), 'labels': probe['labels'], 'logits': [3.0, 1.0]}\n" + + change + + "\nprint(json.dumps(value))\n" + ) + binary.chmod(0o700) + return base, binary + + +def model(base, binary, adapter=None, tokenizer=None): + return NativeDecisionModel( + binary, base, "a" * 40, adapter, tokenizer=tokenizer or Tokenizer() + ) + + +def test_native_subprocess_score_and_backend_specific_calibration(tmp_path): + base, binary = fixture(tmp_path) + native = model(base, binary) + scores = [native.score(item(f"calibration-{i}")) for i in range(4)] + primary = [native.score(item("primary"))] + regression = [native.score(item("regression"))] + fitted = calibrate(scores, primary + regression) + assert metrics(primary, fitted)["accuracy"] == 1 + assert metrics(regression, fitted)["accuracy"] == 1 + assert scores[0]["logits"] == [3.0, 1.0] + assert scores[0]["runtime_identity"]["source_revision_declared"] == "a" * 40 + assert scores[0]["runtime_identity"]["artifacts_observed"][ + "binary_sha256" + ] == file_hash(binary) + assert scores[0]["probe_sha256"] == digest( + {"tokens": [1, 2, 3], "labels": [32, 33]} + ) + with pytest.raises(ValueError, match="overlap"): + metrics(scores, fitted) + with pytest.raises(ValueError, match="another model"): + prediction({**primary[0], "model_identity": "mlx-source"}, fitted) + changed_revision = NativeDecisionModel( + binary, base, "b" * 40, tokenizer=Tokenizer() + ) + with pytest.raises(ValueError, match="another model"): + prediction(changed_revision.score(item()), fitted) + + +@pytest.mark.parametrize("filename", ["model.safetensors", "tokenizer.json"]) +def test_native_scoring_rejects_changed_artifacts(tmp_path, filename): + base, binary = fixture(tmp_path) + native = model(base, binary) + (base / filename).write_bytes(b"changed") + with pytest.raises(ValueError, match="artifacts changed"): + native.score(item()) + + +def test_native_scoring_rejects_change_during_the_actual_process(tmp_path): + change = "Path(sys.argv[sys.argv.index('--hf') + 1], 'model.safetensors').write_bytes(b'changed')\n" + base, binary = fixture(tmp_path, change) + with pytest.raises(ValueError, match="artifacts changed"): + model(base, binary).score(item()) + + +def test_native_timeout_reaps_the_owned_process(tmp_path): + pid_file = tmp_path / "native.pid" + change = ( + f"import os, time\nPath({str(pid_file)!r}).write_text(str(os.getpid()))\n" + "time.sleep(5)\n" + ) + base, binary = fixture(tmp_path, change) + native = NativeDecisionModel( + binary, base, "a" * 40, budget=Budget(0.5), tokenizer=Tokenizer() + ) + with pytest.raises(subprocess.TimeoutExpired): + native.score(item()) + with pytest.raises(ProcessLookupError): + os.kill(int(pid_file.read_text()), 0) + + +@pytest.mark.parametrize( + "change", + [ + "value['labels'] = [33, 32]", + "value['tokens'] = 4", + "value['logits'] = [1.0]", + "value['logits'] = [float('nan'), 1.0]", + "value['logits'] = [True, 1.0]", + ], +) +def test_native_scoring_rejects_mismatched_response(tmp_path, change): + base, binary = fixture(tmp_path, change) + with pytest.raises(ValueError, match="response does not match"): + model(base, binary).score(item()) + + +def test_native_adapter_training_provenance_and_calibration_leakage(tmp_path): + base, binary = fixture(tmp_path) + adapter = tmp_path / "native.lora" + adapter.write_bytes(b"adapter-fixture") + sidecar = Path(str(adapter) + ".json") + metadata = { + "schema": "decision-native-adapter-v1", + "base_files": base_files(base), + "native_adapter_sha256": file_hash(adapter), + } + sidecar.write_text(json.dumps(metadata)) + with pytest.raises(ValueError, match="training provenance"): + model(base, binary, adapter) + metadata["train_identities"] = [identity(item("train"))] + sidecar.write_text(json.dumps(metadata)) + native = model(base, binary, adapter) + trained = native.score(item("train")) + assert trained["seen_in_training"] is True + with pytest.raises(ValueError, match="training examples"): + calibrate([trained]) + assert native.model_identity != model(base, binary).model_identity + adapter.write_bytes(b"changed-adapter") + with pytest.raises(ValueError, match="provenance mismatch"): + model(base, binary, adapter) + + +@pytest.mark.parametrize( + "encoding, message", [([1, 2], "exactly one token"), ([32], "collide")] +) +def test_native_option_token_contract(tmp_path, encoding, message): + base, binary = fixture(tmp_path) + + class InvalidTokenizer(Tokenizer): + def encode(self, text, **kwargs): + return encoding + + with pytest.raises(ValueError, match=message): + model(base, binary, tokenizer=InvalidTokenizer()).score(item()) From ceadfce41c52ba014cbbdfc44615918422cb701b Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Thu, 29 Oct 2026 15:36:06 +0530 Subject: [PATCH 2/6] Reject native calibration leakage and changed tokenizer artifacts --- docs/factory/specialist-labs.md | 6 ++ scripts/specialists/decision.py | 4 +- scripts/specialists/decision_native_score.py | 94 ++++++++++++++++++-- tests/test_specialist_native_scoring.py | 78 ++++++++++++++++ 4 files changed, 172 insertions(+), 10 deletions(-) diff --git a/docs/factory/specialist-labs.md b/docs/factory/specialist-labs.md index 91ff0981..21b2aa2f 100644 --- a/docs/factory/specialist-labs.md +++ b/docs/factory/specialist-labs.md @@ -349,6 +349,12 @@ build provenance. Every exact-token response must match its request. Changed files, unknown adapter training provenance, malformed output and expired budgets fail closed. Native adapter sidecars exported before training identities were preserved require re-export from their original source adapter. +Merged bases retain their own training exclusions too. Standalone tokenizer +files and the resolved chat template are part of this identity. Checksums run at +the start and before final output; per-row file fingerprints detect intervening +changes. Every row cold-loads the native model in a separate bounded process. +The total time budget includes those loads; failure produces no final score +file. Use a small split and retain failed-run diagnostics separately. Use `decision.py calibrate` on native calibration scores only, with train, primary and regression scores supplied through `--exclude`; score those splits diff --git a/scripts/specialists/decision.py b/scripts/specialists/decision.py index 4ad11754..0f74c7dd 100644 --- a/scripts/specialists/decision.py +++ b/scripts/specialists/decision.py @@ -592,7 +592,9 @@ def main(): model = NativeDecisionModel( args.binary, args.base, args.source_revision, args.adapter, budget ) - write_rows(args.out, [model.score(row) for row in rows]) + scores = [model.score(row) for row in rows] + model.finish() + write_rows(args.out, scores) print(json.dumps(budget.measurements())) elif args.command == "export-native": from decision_native import export diff --git a/scripts/specialists/decision_native_score.py b/scripts/specialists/decision_native_score.py index 8db61fb5..716f3348 100644 --- a/scripts/specialists/decision_native_score.py +++ b/scripts/specialists/decision_native_score.py @@ -34,6 +34,9 @@ def __init__( raise ValueError("native binary must be an existing executable file") self.training_identities = set() self.artifacts = self.snapshot() + self.fingerprints = self.file_fingerprints() + if (self.base / "specialist.json").exists(): + self.read_training_identities(read_json(self.base / "specialist.json")) self.check_adapter() if tokenizer is None: from transformers import AutoTokenizer @@ -60,6 +63,9 @@ def __init__( "tokenizer_class": type(tokenizer).__module__ + "." + type(tokenizer).__qualname__, + "resolved_chat_template_sha256": digest( + getattr(tokenizer, "chat_template", None) + ), } self.model_identity = digest(self.runtime_identity) @@ -69,23 +75,78 @@ def snapshot(self): name.endswith(".safetensors") for name in files ): raise ValueError("native base requires config and local model weights") - result = {"binary_sha256": file_hash(self.binary), "base_files": files} + result = { + "binary_sha256": file_hash(self.binary), + "base_files": files, + "tokenizer_files": { + str(path.relative_to(self.base)): file_hash(path) + for path in self.tokenizer_files() + }, + } if self.adapter: result["adapter_sha256"] = file_hash(self.adapter) result["adapter_provenance_sha256"] = file_hash(str(self.adapter) + ".json") return result + def tokenizer_files(self): + return sorted( + { + path + for path in self.base.iterdir() + if path.is_file() + and path.suffix in (".jinja", ".txt", ".model", ".tiktoken") + } + | set((self.base / "chat_templates").glob("*.jinja")) + ) + + def file_fingerprints(self): + paths = {"binary": self.binary} + paths.update( + { + "base/" + path.name: path + for path in self.base.iterdir() + if path.is_file() and path.suffix in (".json", ".safetensors") + } + ) + paths.update( + { + "base/" + str(path.relative_to(self.base)): path + for path in self.tokenizer_files() + } + ) + if self.adapter: + paths.update( + { + "adapter": self.adapter, + "adapter_provenance": Path(str(self.adapter) + ".json"), + } + ) + result = {} + for name, path in paths.items(): + status = path.stat() + result[name] = ( + status.st_dev, + status.st_ino, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + return result + def check_adapter(self): if not self.adapter: return metadata = read_json(str(self.adapter) + ".json") - identities = metadata.get("train_identities") if ( metadata.get("schema") != "decision-native-adapter-v1" or metadata.get("base_files") != self.artifacts["base_files"] or metadata.get("native_adapter_sha256") != self.artifacts["adapter_sha256"] ): raise ValueError("native adapter/base provenance mismatch") + self.read_training_identities(metadata) + + def read_training_identities(self, metadata): + identities = metadata.get("train_identities") if ( not isinstance(identities, list) or not identities @@ -95,13 +156,17 @@ def check_adapter(self): ) or len(set(identities)) != len(identities) ): - raise ValueError( - "native adapter training provenance missing or invalid; re-export" - ) - self.training_identities = set(identities) + raise ValueError("native training provenance missing or invalid; re-export") + self.training_identities.update(identities) def unchanged(self): self.budget.check() + if self.file_fingerprints() != self.fingerprints: + raise ValueError("native scoring artifacts changed; discard this score run") + self.budget.check() + + def finish(self): + self.unchanged() if self.snapshot() != self.artifacts: raise ValueError("native scoring artifacts changed; discard this score run") self.budget.check() @@ -141,11 +206,22 @@ def score(self, item): remaining = self.budget.seconds - (time.monotonic() - self.budget.started) if remaining <= 0: raise TimeoutError("specialist work budget exhausted") - result = subprocess.run( - command, capture_output=True, text=True, timeout=remaining, check=True - ) + try: + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=remaining, + check=True, + ) + except subprocess.CalledProcessError as error: + raise ValueError( + f"native exact-token probe failed ({error.returncode}): {error.stderr.strip()[-1000:]}" + ) from None self.unchanged() value = json.loads(result.stdout) + if not isinstance(value, dict): + raise ValueError("native exact-token response does not match the request") logits = value.get("logits") if ( value.get("schema") != "native-exact-token-logits-v1" diff --git a/tests/test_specialist_native_scoring.py b/tests/test_specialist_native_scoring.py index bfc64a1c..596cd8c1 100644 --- a/tests/test_specialist_native_scoring.py +++ b/tests/test_specialist_native_scoring.py @@ -159,6 +159,84 @@ def test_native_adapter_training_provenance_and_calibration_leakage(tmp_path): model(base, binary, adapter) +def test_native_merged_base_preserves_training_exclusion(tmp_path): + base, binary = fixture(tmp_path) + (base / "specialist.json").write_text( + json.dumps({"train_identities": [identity(item("train"))]}) + ) + native = model(base, binary) + with pytest.raises(ValueError, match="training examples"): + calibrate([native.score(item("train"))]) + + +def test_native_identity_binds_standalone_chat_template(tmp_path): + base, binary = fixture(tmp_path) + old = model(base, binary) + (base / "chat_template.jinja").write_text("changed prompt template") + with pytest.raises(ValueError, match="artifacts changed"): + old.score(item()) + assert model(base, binary).model_identity != old.model_identity + + +def test_native_non_object_response_and_stderr_are_reported(tmp_path): + base, binary = fixture(tmp_path, "value = []") + with pytest.raises(ValueError, match="response does not match"): + model(base, binary).score(item()) + binary.write_text( + f"#!{Path(sys.executable).resolve()}\nimport sys\nsys.stderr.write('base identity mismatch')\nsys.exit(2)\n" + ) + with pytest.raises(ValueError, match="base identity mismatch"): + model(base, binary).score(item()) + + +def test_native_full_checksum_is_rechecked_at_finish(tmp_path, monkeypatch): + base, binary = fixture(tmp_path) + native = model(base, binary) + native.score(item()) + changed = {**native.artifacts, "binary_sha256": "changed"} + monkeypatch.setattr(native, "snapshot", lambda: changed) + with pytest.raises(ValueError, match="artifacts changed"): + native.finish() + + +def test_native_cli_does_not_publish_scores_if_final_validation_fails( + tmp_path, monkeypatch +): + import decision + import decision_native_score + + base, binary = fixture(tmp_path) + native = model(base, binary) + data, out = tmp_path / "data.jsonl", tmp_path / "scores.jsonl" + data.write_text(json.dumps(item()) + "\n") + changed = {**native.artifacts, "binary_sha256": "changed"} + monkeypatch.setattr(native, "snapshot", lambda: changed) + monkeypatch.setattr( + decision_native_score, "NativeDecisionModel", lambda *args: native + ) + monkeypatch.setattr( + sys, + "argv", + [ + "decision.py", + "score-native", + "--binary", + str(binary), + "--base", + str(base), + "--source-revision", + "a" * 40, + "--data", + str(data), + "--out", + str(out), + ], + ) + with pytest.raises(ValueError, match="artifacts changed"): + decision.main() + assert not out.exists() + + @pytest.mark.parametrize( "encoding, message", [([1, 2], "exactly one token"), ([32], "collide")] ) From 2a78636e419e4e99afe3574f38ee93659acb6ec8 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Thu, 29 Oct 2026 15:44:14 +0530 Subject: [PATCH 3/6] Bind native kernels and preflight the complete decision split --- docs/factory/specialist-labs.md | 5 +++ scripts/specialists/decision.py | 1 + scripts/specialists/decision_native_score.py | 38 ++++++++++++++++++-- tests/test_specialist_native_scoring.py | 33 +++++++++++++++-- 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/docs/factory/specialist-labs.md b/docs/factory/specialist-labs.md index 21b2aa2f..6ba6a0d7 100644 --- a/docs/factory/specialist-labs.md +++ b/docs/factory/specialist-labs.md @@ -355,6 +355,11 @@ the start and before final output; per-row file fingerprints detect intervening changes. Every row cold-loads the native model in a separate bounded process. The total time budget includes those loads; failure produces no final score file. Use a small split and retain failed-run diagnostics separately. +All rows are checked against the native 256-token cap before any model process. +Colocated Metal libraries, bundles and dynamic libraries are hashed too; this +does not attest external system libraries or drivers. Exports with unknown +training provenance remain loadable diagnostic artifacts; re-exporting such a +source cannot reconstruct its training identities or qualify confidence. Use `decision.py calibrate` on native calibration scores only, with train, primary and regression scores supplied through `--exclude`; score those splits diff --git a/scripts/specialists/decision.py b/scripts/specialists/decision.py index 0f74c7dd..896e3f7e 100644 --- a/scripts/specialists/decision.py +++ b/scripts/specialists/decision.py @@ -592,6 +592,7 @@ def main(): model = NativeDecisionModel( args.binary, args.base, args.source_revision, args.adapter, budget ) + model.preflight(rows) scores = [model.score(row) for row in rows] model.finish() write_rows(args.out, scores) diff --git a/scripts/specialists/decision_native_score.py b/scripts/specialists/decision_native_score.py index 716f3348..fdf1e238 100644 --- a/scripts/specialists/decision_native_score.py +++ b/scripts/specialists/decision_native_score.py @@ -82,12 +82,26 @@ def snapshot(self): str(path.relative_to(self.base)): file_hash(path) for path in self.tokenizer_files() }, + "native_resources": { + str(path.relative_to(self.binary.parent)): file_hash(path) + for path in self.native_resources() + }, } if self.adapter: result["adapter_sha256"] = file_hash(self.adapter) result["adapter_provenance_sha256"] = file_hash(str(self.adapter) + ".json") return result + def native_resources(self): + paths = set(self.binary.parent.glob("*.metallib")) | set( + self.binary.parent.glob("*.dylib") + ) + for bundle in list(self.binary.parent.glob("*.bundle")) + list( + self.binary.parent.glob("*.framework") + ): + paths.update(path for path in bundle.rglob("*") if path.is_file()) + return sorted(path for path in paths if path.is_file()) + def tokenizer_files(self): return sorted( { @@ -108,6 +122,12 @@ def file_fingerprints(self): if path.is_file() and path.suffix in (".json", ".safetensors") } ) + paths.update( + { + "native/" + str(path.relative_to(self.binary.parent)): path + for path in self.native_resources() + } + ) paths.update( { "base/" + str(path.relative_to(self.base)): path @@ -156,7 +176,9 @@ def read_training_identities(self, metadata): ) or len(set(identities)) != len(identities) ): - raise ValueError("native training provenance missing or invalid; re-export") + raise ValueError( + "native training provenance missing or invalid; source must record original training identities" + ) self.training_identities.update(identities) def unchanged(self): @@ -171,8 +193,7 @@ def finish(self): raise ValueError("native scoring artifacts changed; discard this score run") self.budget.check() - def score(self, item): - self.unchanged() + def prepare(self, item): labels = validate_item(item) ids = option_ids(labels, self.tokenizer) tokens = self.tokenizer.encode( @@ -189,6 +210,17 @@ def score(self, item): raise ValueError( "native prompt requires bounded exact token ids; no truncation" ) + return labels, ids, tokens + + def preflight(self, rows): + self.unchanged() + for item in rows: + self.prepare(item) + self.budget.check() + + def score(self, item): + self.unchanged() + labels, ids, tokens = self.prepare(item) started = time.monotonic() with tempfile.TemporaryDirectory(prefix="posttrain-native-probe-") as directory: probe = Path(directory) / "probe.json" diff --git a/tests/test_specialist_native_scoring.py b/tests/test_specialist_native_scoring.py index 596cd8c1..96609960 100644 --- a/tests/test_specialist_native_scoring.py +++ b/tests/test_specialist_native_scoring.py @@ -105,11 +105,11 @@ def test_native_timeout_reaps_the_owned_process(tmp_path): pid_file = tmp_path / "native.pid" change = ( f"import os, time\nPath({str(pid_file)!r}).write_text(str(os.getpid()))\n" - "time.sleep(5)\n" + "time.sleep(30)\n" ) base, binary = fixture(tmp_path, change) native = NativeDecisionModel( - binary, base, "a" * 40, budget=Budget(0.5), tokenizer=Tokenizer() + binary, base, "a" * 40, budget=Budget(3), tokenizer=Tokenizer() ) with pytest.raises(subprocess.TimeoutExpired): native.score(item()) @@ -178,6 +178,35 @@ def test_native_identity_binds_standalone_chat_template(tmp_path): assert model(base, binary).model_identity != old.model_identity +def test_native_identity_binds_colocated_metal_resources(tmp_path): + base, binary = fixture(tmp_path) + bundle = binary.parent / "mlx-swift_Cmlx.bundle" + bundle.mkdir() + shader = bundle / "default.metallib" + shader.write_bytes(b"compiled-metal-fixture") + native = model(base, binary) + shader.write_bytes(b"changed-metal-fixture") + with pytest.raises(ValueError, match="artifacts changed"): + native.score(item()) + assert model(base, binary).model_identity != native.model_identity + + +def test_native_preflight_rejects_late_oversize_before_any_process(tmp_path): + marker = tmp_path / "native-called" + base, binary = fixture(tmp_path, f"Path({str(marker)!r}).write_text('called')") + + class LongTokenizer(Tokenizer): + def encode(self, text, **kwargs): + if "too-long" in text: + return [1] * 257 + return super().encode(text, **kwargs) + + native = model(base, binary, tokenizer=LongTokenizer()) + with pytest.raises(ValueError, match="bounded exact token"): + native.preflight([item("valid"), item("too-long")]) + assert not marker.exists() + + def test_native_non_object_response_and_stderr_are_reported(tmp_path): base, binary = fixture(tmp_path, "value = []") with pytest.raises(ValueError, match="response does not match"): From 36492cc699f0869f39e7531235df5bda8eee8a67 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Thu, 29 Oct 2026 15:50:02 +0530 Subject: [PATCH 4/6] Bind packaged native resources and reject missing Metal artifacts --- docs/factory/specialist-labs.md | 6 +- scripts/specialists/decision_native_score.py | 39 ++++++++----- tests/test_specialist_native_scoring.py | 60 +++++++++++++++++++- 3 files changed, 88 insertions(+), 17 deletions(-) diff --git a/docs/factory/specialist-labs.md b/docs/factory/specialist-labs.md index 6ba6a0d7..40ff35f4 100644 --- a/docs/factory/specialist-labs.md +++ b/docs/factory/specialist-labs.md @@ -356,8 +356,10 @@ changes. Every row cold-loads the native model in a separate bounded process. The total time budget includes those loads; failure produces no final score file. Use a small split and retain failed-run diagnostics separately. All rows are checked against the native 256-token cap before any model process. -Colocated Metal libraries, bundles and dynamic libraries are hashed too; this -does not attest external system libraries or drivers. Exports with unknown +Metal libraries, bundles and dynamic libraries beside the executable or in its +packaged app's `Contents/Resources` and `Contents/Frameworks` are hashed too. +Missing Metal libraries block scoring. This does not attest external system +libraries or drivers. Exports with unknown training provenance remain loadable diagnostic artifacts; re-exporting such a source cannot reconstruct its training identities or qualify confidence. diff --git a/scripts/specialists/decision_native_score.py b/scripts/specialists/decision_native_score.py index fdf1e238..5e2b3c52 100644 --- a/scripts/specialists/decision_native_score.py +++ b/scripts/specialists/decision_native_score.py @@ -75,6 +75,11 @@ def snapshot(self): name.endswith(".safetensors") for name in files ): raise ValueError("native base requires config and local model weights") + resources = self.native_resources() + if not any(path.suffix == ".metallib" for path in resources.values()): + raise ValueError( + "native Metal kernel resources unavailable; retain the CLI's Metal library" + ) result = { "binary_sha256": file_hash(self.binary), "base_files": files, @@ -83,8 +88,7 @@ def snapshot(self): for path in self.tokenizer_files() }, "native_resources": { - str(path.relative_to(self.binary.parent)): file_hash(path) - for path in self.native_resources() + name: file_hash(path) for name, path in resources.items() }, } if self.adapter: @@ -93,14 +97,26 @@ def snapshot(self): return result def native_resources(self): - paths = set(self.binary.parent.glob("*.metallib")) | set( - self.binary.parent.glob("*.dylib") - ) - for bundle in list(self.binary.parent.glob("*.bundle")) + list( - self.binary.parent.glob("*.framework") + roots = {"binary": self.binary.parent} + if ( + self.binary.parent.name == "MacOS" + and self.binary.parent.parent.name == "Contents" ): - paths.update(path for path in bundle.rglob("*") if path.is_file()) - return sorted(path for path in paths if path.is_file()) + roots["app-resources"] = self.binary.parent.parent / "Resources" + roots["app-frameworks"] = self.binary.parent.parent / "Frameworks" + resources = {} + for name, root in roots.items(): + paths = set(root.glob("*.metallib")) | set(root.glob("*.dylib")) + for bundle in list(root.glob("*.bundle")) + list(root.glob("*.framework")): + paths.update(path for path in bundle.rglob("*") if path.is_file()) + resources.update( + { + name + "/" + str(path.relative_to(root)): path + for path in sorted(paths) + if path.is_file() + } + ) + return resources def tokenizer_files(self): return sorted( @@ -123,10 +139,7 @@ def file_fingerprints(self): } ) paths.update( - { - "native/" + str(path.relative_to(self.binary.parent)): path - for path in self.native_resources() - } + {"native/" + name: path for name, path in self.native_resources().items()} ) paths.update( { diff --git a/tests/test_specialist_native_scoring.py b/tests/test_specialist_native_scoring.py index 96609960..aa2a2a62 100644 --- a/tests/test_specialist_native_scoring.py +++ b/tests/test_specialist_native_scoring.py @@ -48,6 +48,7 @@ def fixture(tmp_path, change=""): + "\nprint(json.dumps(value))\n" ) binary.chmod(0o700) + (binary.parent / "default.metallib").write_bytes(b"process-fixture-only-metal") return base, binary @@ -191,7 +192,36 @@ def test_native_identity_binds_colocated_metal_resources(tmp_path): assert model(base, binary).model_identity != native.model_identity -def test_native_preflight_rejects_late_oversize_before_any_process(tmp_path): +def test_native_identity_binds_packaged_app_resources(tmp_path): + base, binary = fixture(tmp_path) + contents = tmp_path / "posttrainllm.app/Contents" + destination = contents / "MacOS/posttrainllm-cli" + destination.parent.mkdir(parents=True) + binary.rename(destination) + bundle = contents / "Resources/mlx-swift_Cmlx.bundle" + bundle.mkdir(parents=True) + shader = bundle / "default.metallib" + shader.write_bytes(b"packaged-metal-fixture") + native = model(base, destination) + shader.write_bytes(b"changed-packaged-metal-fixture") + with pytest.raises(ValueError, match="artifacts changed"): + native.score(item()) + assert model(base, destination).model_identity != native.model_identity + + +def test_native_missing_metal_resources_cannot_qualify_scores(tmp_path): + base, binary = fixture(tmp_path) + (binary.parent / "default.metallib").unlink() + with pytest.raises(ValueError, match="Metal kernel resources unavailable"): + model(base, binary) + + +def test_native_preflight_rejects_late_oversize_before_any_process( + tmp_path, monkeypatch +): + import decision + import decision_native_score + marker = tmp_path / "native-called" base, binary = fixture(tmp_path, f"Path({str(marker)!r}).write_text('called')") @@ -202,9 +232,35 @@ def encode(self, text, **kwargs): return super().encode(text, **kwargs) native = model(base, binary, tokenizer=LongTokenizer()) + data, out = tmp_path / "data.jsonl", tmp_path / "scores.jsonl" + data.write_text( + "\n".join(json.dumps(row) for row in [item("valid"), item("too-long")]) + "\n" + ) + monkeypatch.setattr( + decision_native_score, "NativeDecisionModel", lambda *args: native + ) + monkeypatch.setattr( + sys, + "argv", + [ + "decision.py", + "score-native", + "--binary", + str(binary), + "--base", + str(base), + "--source-revision", + "a" * 40, + "--data", + str(data), + "--out", + str(out), + ], + ) with pytest.raises(ValueError, match="bounded exact token"): - native.preflight([item("valid"), item("too-long")]) + decision.main() assert not marker.exists() + assert not out.exists() def test_native_non_object_response_and_stderr_are_reported(tmp_path): From 0b14d7d2bb860b626d25d6a5a6db3ed56468724c Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Thu, 29 Oct 2026 15:59:18 +0530 Subject: [PATCH 5/6] Bind native prompt rendering library versions --- scripts/specialists/decision_native_score.py | 2 +- tests/test_specialist_native_scoring.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/specialists/decision_native_score.py b/scripts/specialists/decision_native_score.py index 5e2b3c52..13614c12 100644 --- a/scripts/specialists/decision_native_score.py +++ b/scripts/specialists/decision_native_score.py @@ -46,7 +46,7 @@ def __init__( ) self.tokenizer = tokenizer tokenizer_versions = {} - for package in ("transformers", "tokenizers"): + for package in ("transformers", "tokenizers", "jinja2", "markupsafe"): try: tokenizer_versions[package] = importlib.metadata.version(package) except importlib.metadata.PackageNotFoundError: diff --git a/tests/test_specialist_native_scoring.py b/tests/test_specialist_native_scoring.py index aa2a2a62..34de4166 100644 --- a/tests/test_specialist_native_scoring.py +++ b/tests/test_specialist_native_scoring.py @@ -179,6 +179,20 @@ def test_native_identity_binds_standalone_chat_template(tmp_path): assert model(base, binary).model_identity != old.model_identity +def test_native_identity_binds_template_renderer_version(tmp_path, monkeypatch): + import decision_native_score + + base, binary = fixture(tmp_path) + native = model(base, binary) + version = decision_native_score.importlib.metadata.version + monkeypatch.setattr( + decision_native_score.importlib.metadata, + "version", + lambda name: "changed-renderer" if name == "jinja2" else version(name), + ) + assert model(base, binary).model_identity != native.model_identity + + def test_native_identity_binds_colocated_metal_resources(tmp_path): base, binary = fixture(tmp_path) bundle = binary.parent / "mlx-swift_Cmlx.bundle" From 7f735e9d3e4b1aebbdcabcffcf3bb10296d2de57 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Thu, 29 Oct 2026 16:06:52 +0530 Subject: [PATCH 6/6] Keep native command dispatch within the complexity budget --- scripts/specialists/decision.py | 41 +++++++++++++++----------- tests/test_specialist_native_export.py | 27 +++++++++++++++++ 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/scripts/specialists/decision.py b/scripts/specialists/decision.py index 896e3f7e..19fc8473 100644 --- a/scripts/specialists/decision.py +++ b/scripts/specialists/decision.py @@ -580,27 +580,32 @@ def run_model(args): print(json.dumps(budget.measurements())) -def main(): - args = argument_parser().parse_args() - if args.command == "score-native": - from decision_native_score import NativeDecisionModel - - budget = Budget(args.seconds) - rows = read_rows(args.data) - if Path(args.out).exists(): - raise FileExistsError(args.out) - model = NativeDecisionModel( - args.binary, args.base, args.source_revision, args.adapter, budget - ) - model.preflight(rows) - scores = [model.score(row) for row in rows] - model.finish() - write_rows(args.out, scores) - print(json.dumps(budget.measurements())) - elif args.command == "export-native": +def run_native(args): + if args.command == "export-native": from decision_native import export print(json.dumps(export(args.base, args.adapter, args.out))) + return + from decision_native_score import NativeDecisionModel + + budget = Budget(args.seconds) + rows = read_rows(args.data) + if Path(args.out).exists(): + raise FileExistsError(args.out) + model = NativeDecisionModel( + args.binary, args.base, args.source_revision, args.adapter, budget + ) + model.preflight(rows) + scores = [model.score(row) for row in rows] + model.finish() + write_rows(args.out, scores) + print(json.dumps(budget.measurements())) + + +def main(): + args = argument_parser().parse_args() + if args.command in ("score-native", "export-native"): + run_native(args) elif args.command == "export-browser": from decision_browser import export diff --git a/tests/test_specialist_native_export.py b/tests/test_specialist_native_export.py index aba1e010..5d34f769 100644 --- a/tests/test_specialist_native_export.py +++ b/tests/test_specialist_native_export.py @@ -99,6 +99,33 @@ def test_native_export_rejects_changed_base_or_adapter(tmp_path): export(base, adapter, tmp_path / "bad.lora") +def test_native_export_operator_command_preserves_identity( + tmp_path, monkeypatch, capsys +): + import decision + + base, adapter = fixture(tmp_path) + out = tmp_path / "operator.lora" + monkeypatch.setattr( + sys, + "argv", + [ + "decision.py", + "export-native", + "--base", + str(base), + "--adapter", + str(adapter), + "--out", + str(out), + ], + ) + decision.main() + receipt = json.loads(capsys.readouterr().out) + assert receipt["native_adapter_sha256"] == file_hash(out) + assert receipt["train_identities"] == [digest("training-example")] + + def test_native_float32_conversion_rejects_overflow(tmp_path): from safetensors.numpy import load_file