diff --git a/.github/scripts/download_verified_artifact.ps1 b/.github/scripts/download_verified_artifact.ps1 new file mode 100644 index 00000000..376c675e --- /dev/null +++ b/.github/scripts/download_verified_artifact.ps1 @@ -0,0 +1,48 @@ +param( + [Parameter(Mandatory = $true)] + [uri] $Url, + [Parameter(Mandatory = $true)] + [string] $Output, + [Parameter(Mandatory = $true)] + [ValidatePattern('^[A-Fa-f0-9]{64}$')] + [string] $Sha256 +) + +$ErrorActionPreference = "Stop" +if ($Url.Scheme -ne "https" -and -not ($Url.Scheme -eq "http" -and $Url.IsLoopback)) { + throw "Artifact URL must use HTTPS." +} + +$maximumAttempts = 3 +$partialOutput = "$Output.$PID.partial" +$downloaded = $false +for ($attempt = 1; $attempt -le $maximumAttempts; $attempt++) { + try { + Invoke-WebRequest -Uri $Url -OutFile $partialOutput -TimeoutSec 120 | Out-Null + $downloaded = $true + break + } + catch { + Remove-Item -LiteralPath $partialOutput -Force -ErrorAction SilentlyContinue + $statusCode = 0 + $response = $_.Exception.Response + if ($null -ne $response -and $null -ne $response.StatusCode) { + $statusCode = [int] $response.StatusCode + } + $transientStatus = $statusCode -eq 0 -or $statusCode -in @(408, 429) -or ($statusCode -ge 500 -and $statusCode -lt 600) + if (-not $transientStatus -or $attempt -eq $maximumAttempts) { + throw + } + Start-Sleep -Seconds $attempt + } +} + +if (-not $downloaded) { + throw "Artifact download failed after $maximumAttempts attempts." +} +$actualSha256 = (Get-FileHash -Algorithm SHA256 -Path $partialOutput).Hash.ToLowerInvariant() +if ($actualSha256 -ne $Sha256.ToLowerInvariant()) { + Remove-Item -LiteralPath $partialOutput -Force + throw "Artifact SHA-256 mismatch: expected $Sha256, got $actualSha256" +} +Move-Item -LiteralPath $partialOutput -Destination $Output -Force diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 365ae0dd..79d005eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,6 +109,7 @@ jobs: unit-tests: name: Unit Tests runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Checkout uses: actions/checkout@v5 @@ -203,7 +204,9 @@ jobs: - name: Run product smoke tests run: | python -W error -m pytest -v tests/product_smoke \ - -m "stable" --no-cov --tb=short + -m "stable" --no-cov --tb=short \ + --junitxml="$RUNNER_TEMP/stable-product-smoke.xml" + python scripts/check_critical_skips.py "$RUNNER_TEMP/stable-product-smoke.xml" cross-platform-tests: name: Tests (${{ matrix.os }}, Python ${{ matrix.python-version }}) @@ -230,7 +233,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y \ - git build-essential gcc-arm-linux-gnueabihf qemu-user pkg-config wget cmake + git build-essential clang gcc-arm-linux-gnueabihf qemu-user pkg-config wget cmake - name: Install system dependencies (macOS) if: runner.os == 'macOS' @@ -252,11 +255,7 @@ jobs: $archive = Join-Path $env:RUNNER_TEMP $archiveName $url = "https://github.com/radareorg/radare2/releases/download/$version/$archiveName" $expectedSha256 = "913e7d95e7458226a5e783240877f2c1d405398ab38ba176ce9b12e046e31f34" - Invoke-WebRequest -Uri $url -OutFile $archive - $actualSha256 = (Get-FileHash -Algorithm SHA256 -Path $archive).Hash.ToLowerInvariant() - if ($actualSha256 -ne $expectedSha256) { - throw "radare2 archive checksum mismatch: expected $expectedSha256, got $actualSha256" - } + & ".github/scripts/download_verified_artifact.ps1" -Url $url -Output $archive -Sha256 $expectedSha256 $destination = Join-Path $env:RUNNER_TEMP "radare2-$version" Expand-Archive -Path $archive -DestinationPath $destination -Force $bin = Join-Path $destination "radare2-$version-w64\bin" @@ -320,6 +319,7 @@ jobs: test_targets = [ "tests/unit/test_circular_imports.py", "tests/unit/test_cli_basic_commands.py::test_cli_version_function", + "tests/unit/test_public_surface_contract.py", "tests/unit/test_pe_handler_parsing_contract.py", "tests/unit/test_pe_handler_struct_parse.py", "tests/unit/test_macho_parse_basic.py", @@ -438,4 +438,5 @@ jobs: /tmp/r2morph-wheel-venv/bin/python -W error -m pytest \ tests/unit/test_circular_imports.py \ tests/unit/test_cli_basic_commands.py::test_cli_version_function \ + tests/unit/test_public_surface_contract.py \ --no-cov --tb=short diff --git a/.github/workflows/differential-corpus.yml b/.github/workflows/differential-corpus.yml index 49d18baf..01eacc66 100644 --- a/.github/workflows/differential-corpus.yml +++ b/.github/workflows/differential-corpus.yml @@ -11,6 +11,7 @@ concurrency: - ".github/workflows/differential-corpus.yml" - ".github/workflows/ci.yml" - ".github/workflows/adversarial-benchmark.yml" + - ".github/scripts/download_verified_artifact.ps1" - ".github/scripts/install_radare2.sh" - "docs/compatibility-corpus.md" - "docs/pass-maturity.md" @@ -1612,11 +1613,7 @@ jobs: $archive = Join-Path $env:RUNNER_TEMP $archiveName $url = "https://github.com/radareorg/radare2/releases/download/$version/$archiveName" $expectedSha256 = "913e7d95e7458226a5e783240877f2c1d405398ab38ba176ce9b12e046e31f34" - Invoke-WebRequest -Uri $url -OutFile $archive - $actualSha256 = (Get-FileHash -Algorithm SHA256 -Path $archive).Hash.ToLowerInvariant() - if ($actualSha256 -ne $expectedSha256) { - throw "radare2 archive checksum mismatch: expected $expectedSha256, got $actualSha256" - } + & ".github/scripts/download_verified_artifact.ps1" -Url $url -Output $archive -Sha256 $expectedSha256 $destination = Join-Path $env:RUNNER_TEMP "radare2-$version" Expand-Archive -Path $archive -DestinationPath $destination -Force $bin = Join-Path $destination "radare2-$version-w64\bin" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1e303942..ddf1c7ad 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,6 +28,11 @@ jobs: with: python-version: "3.12" + - name: Install release dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Extract version from tag id: get_version run: echo "version=${GITHUB_REF#refs/tags/v}" >> "$GITHUB_OUTPUT" diff --git a/CHANGELOG.md b/CHANGELOG.md index 0804e2ec..3456b12b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## 0.4.0-alpha.3 +- Clarifies campaign-specific IDA and Triton availability and limits VM resistance + claims to the finite evidence actually measured. - Promotes instruction expansion and block reordering into the five-pass Tier 1 Linux ELF x86-64 contract after native, corpus, composition, and analyzer evidence completed. @@ -13,7 +15,7 @@ documented wheel gate runs from a normal development installation. - Fixes release artifact download in the isolated PyPI publish job. - Publishes the same verified ELF x86-64 support contract as alpha.2. -- Keeps the VM milestone blocked until external human review records signoff. +- Independent human review approval is recorded for the VM resistance campaign. ## 0.4.0-alpha.2 diff --git a/README.md b/README.md index c5292947..5242e877 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,8 @@ x86-64 or equivalent CodeVirtualization support. The machine-readable contract is [`docs/support-matrix.json`](docs/support-matrix.json): the generated matrix currently records 22/22 evidenced cells for the official -ELF x86-64 target (100.0% evidence) and 32/242 evidenced cells for non-official -targets (13.22% evidence). It currently classifies 5 passes as Tier 1 and +ELF x86-64 target (100.0% evidence) and 35/242 evidenced cells for non-official +targets (14.46% evidence). It currently classifies 5 passes as Tier 1 and 17 passes as experimental. The scheduled extended maturity smoke now records output-size, transform-duration, runtime-duration, and static metric summaries for every @@ -174,13 +174,12 @@ transform duration, runtime duration, and static analyzer evidence while comparing exit code, stdout, stderr, created files, and declared observable effects across nine seed-derived inputs. -The latest local adversarial refresh measures all 162 repository fixtures: +The 2026-09-20 local adversarial campaign measures all 162 repository fixtures: `angr` completed 162/162 runs, while radare2, objdump, and the custom analyzer also completed all runs. Unicorn completed 155/162 runs; its seven unavailable -rows are retained with their instruction-support reasons. Binary Ninja, IDA, -and Triton remain explicit unavailable analyzer slots; Binary Ninja is on hold -for the current milestone. Ghidra is now provisioned by the scheduled -campaign. The bounded +rows are retained with their instruction-support reasons. In that campaign, +IDA and Triton were unavailable; these rows remain explicit and are not counted +as completed analysis. Ghidra is provisioned by the scheduled campaign. The bounded report is [`docs/protection-adversarial-angr-local-2026-09-20.json`](docs/protection-adversarial-angr-local-2026-09-20.json). The earlier focused `angr` fixture remains available in @@ -190,23 +189,38 @@ Binary Ninja is an explicit analyzer slot; the focused local fixture records `binary-ninja` as unavailable because `module 'binaryninja' is unavailable`. The current installed API is importable, but the headless benchmark reports `analyzer license is unavailable`; this remains an explicit evidence gap while -the slot is on hold. -The VM review packet still has `human_signoff: not-attested` and -`release_decision: block-vm-milestone`; memory, direct/indirect calls, returns, -flags, FP/SIMD, varargs/ABI, unwinding, TLS/signals, SSA, and liveness paths -remain explicit review scope, and unsupported instructions must fail closed -instead of producing partial protected functions. Unsupported-instruction -diagnostics must include instruction address, mnemonic, type, size, bounded -opcode preview, capability, and reason. -VM ISA/opcode diversification, dispatcher/handler alternatives, -superinstructions, anti-tamper, and progressive bytecode protection remain -explicit human-review scope before the virtualizer milestone can be marked -complete. +the slot is on hold. Separately, the 159-sample evidence summarized in +[`docs/independent-review.json`](docs/independent-review.json) records +`159 samples, 159 Triton runs`, `159 samples, 318 completed analyses` for IDA, +and `159 samples, 0 IDA errors`. These are campaign-specific +results: availability in one campaign does not fill unavailable rows in another, +and the all-named-analyzers 1.0 criterion remains open. +The VM review packet records `human_signoff: attested` and +`release_decision: signoff-recorded`. Human independent review approved the +reviewed commit `a111f923df43a66826b71ab70db8a484532fbf3b`. The project owner +confirms Marc Rivero Lopez's independence and specialist experience in binary +rewriting and virtualization. He reviewed memory, direct/indirect calls, +returns, flags, FP/SIMD, varargs/ABI, +unwinding, TLS/signals, SSA, and liveness paths; unsupported instructions must +fail closed instead of producing partial protected functions. Diagnostics +include instruction address, mnemonic, type, size, bounded opcode preview, +capability, and reason. The review covered VM ISA/opcode diversification, +dispatcher/handler alternatives, superinstructions, anti-tamper, and progressive +bytecode protection, fuzz properties, failure handling, and bounded report +data. This approval applies to that commit; changes to the reviewed VM +implementation require a new independent review. Current VM resistance artifacts cover 10 seeds with 255 handlers per seed, 0 exact normalized cross-seed handler matches, 12 handler stride values, and target handler stride diversity recorded in [`docs/protection-handler-clustering.json`](docs/protection-handler-clustering.json) and [`docs/protection-bytecode-grammar.json`](docs/protection-bytecode-grammar.json). +These finite seed-diversity measurements describe generated output variation; +they do not measure generalized attack cost or establish resistance to +professional reverse engineering. The independent analysis report explicitly +limits its evidence to one fixture and one pass and states that it does not +establish general resistance to professional reverse engineering. The report +does not replace the required three-analyzer release campaign. Published +resistance results remain an open 1.0 criterion. --- diff --git a/docs/API.md b/docs/API.md index d2ca0b75..0e1e292f 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,5 +1,11 @@ # r2morph API Documentation +> **Compatibility scope:** This guide includes module-level examples that are +> useful for development but are not all stable imports. The intended 1.0 +> compatibility surface and deprecation policy are defined in +> [stability-contract.md](stability-contract.md); currently the package-root +> exports and documented CLI surface are the contracted entry points. + ## Overview r2morph is a metamorphic mutation engine for binary transformation. The API is organized into several modules: diff --git a/docs/compatibility-corpus.md b/docs/compatibility-corpus.md index bb227b2c..bd39cbb9 100644 --- a/docs/compatibility-corpus.md +++ b/docs/compatibility-corpus.md @@ -38,6 +38,22 @@ compiled case is an additional evidence point, not a support promotion. The Windows PE preview job also executes a real NOP-plus-register-substitution composition and validates the repaired PE checksum; it remains preview evidence and does not promote PE to the official support tier. +On 2026-11-10, the same PE test module ran locally on macOS 27.2 ARM64 with +MinGW-w64 GCC 16.2 and Wine 11.0: seven generated-binary mutation tests passed +and the Windows-only fixture test was skipped. This confirms an additional +Wine execution path; it is not native Windows evidence or a PE parity claim. +The PE block-reordering regression now builds a separate control-flow fixture, +executes the original and mutated images under native Windows or Wine for nine +inputs, and validates the repaired PE checksum and integrity. A passing local +run is preview evidence for this pass/target cell only; it does not promote +PE x86-64 to parity with Linux ELF x86-64. +On 2026-11-10, a connected licensed static-analysis service inspected the +original and CodeVirtualization-protected `elf_x86_64` fixture. The bounded +function and decompilation observations are in +[`protection-independent-analysis-2026-11-10.json`](protection-independent-analysis-2026-11-10.json). +The service did not expose its version; runtime equivalence was not measured. +This one-fixture result does not establish general resistance or replace the +three-analyzer release campaign. The cross-format matrix additionally exercises `basic` and `complex` CFG variants for those three ELF targets across the same four individual passes. The workflow pins the collected matrix at 73 cases so loss of a preview corpus diff --git a/docs/human-review-signoff.json b/docs/human-review-signoff.json new file mode 100644 index 00000000..cc7fb097 --- /dev/null +++ b/docs/human-review-signoff.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "reviewer": "Marc Rivero Lopez", + "review_date_utc": "2026-11-08", + "result": "APPROVE", + "reviewed_commit": "a111f923df43a66826b71ab70db8a484532fbf3b", + "independent": true, + "independence_confirmation": "confirmed by the project owner on 2026-11-09", + "specialist_expertise_confirmation": "binary rewriting and virtualization experience confirmed by the project owner on 2026-11-09", + "findings": [] +} diff --git a/docs/independent-review-packet.md b/docs/independent-review-packet.md index 53ec8fe6..e1ba9ac0 100644 --- a/docs/independent-review-packet.md +++ b/docs/independent-review-packet.md @@ -1,15 +1,13 @@ # Independent Review Packet -This packet defines the human review required before the current virtualizer -milestone can be marked complete. It is intentionally unsigned: automated -checks are evidence, not a human approval. +This packet records the completed independent human review of VM resistance +evidence. Automated checks remain separate evidence and do not replace the +approval recorded below. ## Target -- Review the exact pushed `main` HEAD when the human review begins; record its - full 40-character commit SHA in the sign-off below. This keeps the review - bound to the current code instead of the historical commit listed in the - earlier agent review record. +- Reviewed commit: `a111f923df43a66826b71ab70db8a484532fbf3b`. The approval + below applies to this exact commit. - Original production evidence baseline: `9ac3e14`; inspect the current source and all relevant changes since that baseline. - Scope: ELF x86-64 CodeVirtualization, VM diversification, analyzer corpus, @@ -67,8 +65,8 @@ exit `6 -> 6`, IDA with `1/2` functions and zero errors, and Triton with `8/8` original instructions semantically supported. Binary Ninja is measured by the benchmark when its licensed API is available; older artifacts that predate that integration remain historical evidence only. -The automated review still reports `human_signoff: not-attested` and -`release_decision: block-vm-milestone`. +At that historical rerun, the automated review reported `human_signoff: +not-attested` and `release_decision: block-vm-milestone`. The current-state IDA MCP rerun for the named regression is recorded in the `a3b8c6e` artifact: one original function, nine protected functions, and zero analysis errors. Native ELF execution was not attempted on macOS. @@ -91,8 +89,9 @@ functions, zero unsupported or partial virtualizations, 939 completed analyzer runs, 15 isolated Unicorn errors, and 318 explicit local IDA/Ghidra-unavailable rows. Angr and Triton complete every original/protected pair. This raw rerun predates the Binary Ninja analyzer slot; the live benchmark contract now keeps -that slot explicit when its licensed API is available or unavailable. Human -signoff remains `not-attested`. +that slot explicit when its licensed API is available or unavailable. At that +historical rerun, human signoff was `not-attested`; the independent approval +recorded below is later. The pushed CI workflow `34065194410` completed successfully across its Linux, macOS, and Windows matrix, including installed-wheel smoke tests, typecheck, lint, integration, and property/fuzz validation. @@ -113,7 +112,8 @@ resistance scope, the scheduled adversarial continuous-evidence gate, the `angr`/Binary Ninja analyzer availability split, and the existing 159-sample virtualization corpus. The same automated review verifies that unavailable analyzer slots, including -Binary Ninja, remain release-signoff blockers instead of completed evidence. +Binary Ninja, remain explicit unavailable evidence instead of completed +evidence. The target history extends the compiler-generated ELF regression outside that corpus to GCC fixed-load `-O0`, `-O1`, `-O2`, `-O3`, and `-Os`, GCC fixed-load `-O2` with frame pointers, GCC PIE `-O2`, GCC stripped fixed-load `-O2`, and @@ -128,10 +128,11 @@ integer operations and comparisons through `imul`, `neg`, `not`, `cmp`, `sete`, and `movzx`, as well as carry consumption and rotate round-trips through `stc`, `adc`, `clc`, `sbb`, `rol`, and `ror`. CI run `34173770557` completed successfully across the configured matrix. -The same rerun reports `human_signoff: not-attested`; this packet therefore -remains technically updated but not human-approved. The release contract at -this commit also validates that the packet and compatibility documentation do -not contain broken local Markdown links. Release `v0.4.0-alpha.1` is published +That historical rerun reported `human_signoff: not-attested`. The later +independent approval recorded below supersedes that status. The release +contract at this commit also validates that the packet and compatibility +documentation do not contain broken local Markdown links. Release +`v0.4.0-alpha.1` is published with wheel, source archive, SBOM, checksums, and verified provenance; this is release evidence, not human approval. @@ -212,8 +213,8 @@ core, installed-wheel, and platform jobs. ARM64 remains experimental. The current target records the generated ARM64 register-substitution evidence in the exhaustive support matrix. The Mach-O/AArch64 cell is marked `evidenced`, while the platform remains experimental and outside the official -Linux ELF x86-64 guarantee. The matrix still contains 264 explicit cells and -the automated review continues to require human signoff. +Linux ELF x86-64 guarantee. The matrix still contains 264 explicit cells. At +that historical review point, the automated report still required human signoff. The current target also adds a regression contract that rejects `supported` status for every non-official format or architecture. PE, Mach-O, ARM, and @@ -240,24 +241,28 @@ the aggregate counters alone. ## Human checklist -- [ ] Confirm the `elf_vm_fppackedidxnb_x86_64` regression is generic and not +- [x] Confirm the `elf_vm_fppackedidxnb_x86_64` regression is generic and not sample-specific. -- [ ] Inspect memory, direct/indirect calls, returns, flags, FP/SIMD, +- [x] Inspect memory, direct/indirect calls, returns, flags, FP/SIMD, varargs/ABI, unwinding, TLS/signals, SSA, and liveness paths. -- [ ] Confirm unsupported instructions fail closed and cannot silently produce +- [x] Confirm unsupported instructions fail closed and cannot silently produce a partial protected function. -- [ ] Confirm unsupported-instruction diagnostics include instruction address, +- [x] Confirm unsupported-instruction diagnostics include instruction address, mnemonic, type, size, bounded opcode preview, capability, and reason. -- [ ] Review VM ISA/opcode diversification, dispatcher/handler alternatives, +- [x] Review VM ISA/opcode diversification, dispatcher/handler alternatives, superinstructions, anti-tamper, and progressive bytecode protection. -- [ ] Review the fuzz properties and failure handling for dispatcher, +- [x] Review the fuzz properties and failure handling for dispatcher, relocations, and rewriting. -- [ ] Check that reports contain bounded metadata rather than raw sample bytes +- [x] Check that reports contain bounded metadata rather than raw sample bytes or analyst host paths. -- [ ] Record any finding with a reproducer, severity, and affected commit. +- [x] Record any finding with a reproducer, severity, and affected commit; no + findings were reported. ## Sign-off +Current automated report: `human_signoff: attested`; +`release_decision: signoff-recorded`. + ### Historical agent review record (not current human sign-off) Reviewer: Codex (AI coding agent) @@ -268,15 +273,18 @@ Independence statement: This is an AI-assisted repository review, not a human independent review or approval. The reviewer has no authority to attest human independence on behalf of a person. -```text -Reviewer: -Affiliation: -Date (UTC): -Independence statement: -Result: APPROVE / APPROVE WITH FINDINGS / REJECT -Findings: -Reviewed commit (full 40-character SHA): -``` +### Independent human review + +Reviewer: Marc Rivero Lopez (identity confirmed by the project owner) +Affiliation: Not provided +Date (UTC): 2026-11-08 +Result: APPROVE +Findings: None reported +Reviewed commit: `a111f923df43a66826b71ab70db8a484532fbf3b` +Independence statement: The project owner confirmed on 2026-11-09 that the +reviewer acted independently of the authoring team and has specialist +experience in binary rewriting and virtualization. The signoff is recorded +from that explicit confirmation and is not cryptographically signed. -No approval is implied until a human reviewer fills this section outside the -automated test process. +This approval applies to the reviewed commit only. Changes to the reviewed VM +implementation require a new independent review. diff --git a/docs/independent-review.json b/docs/independent-review.json index df0baf75..8fb5296b 100644 --- a/docs/independent-review.json +++ b/docs/independent-review.json @@ -16,7 +16,7 @@ "status": "passed" }, { - "detail": "release blocker ledger tracks per-pass, differential, VM, parity, benchmark, resistance, and signoff gaps", + "detail": "release blocker ledger tracks per-pass, differential, VM, parity, benchmark, resistance, and review signoff", "name": "release_blocker_ledger_scope", "status": "passed" }, @@ -51,7 +51,7 @@ "status": "passed" }, { - "detail": "VM resistance remains pending human adversarial review", + "detail": "automated resistance evidence is paired with an independent human signoff", "name": "vm_resistance_adversarial_scope", "status": "passed" }, @@ -126,14 +126,25 @@ "status": "passed" } ], - "human_signoff": "not-attested", + "human_review": { + "findings": [], + "independence_confirmation": "confirmed by the project owner on 2026-11-09", + "independent": true, + "result": "APPROVE", + "review_date_utc": "2026-11-08", + "reviewed_commit": "a111f923df43a66826b71ab70db8a484532fbf3b", + "reviewer": "Marc Rivero Lopez", + "schema_version": 1, + "specialist_expertise_confirmation": "binary rewriting and virtualization experience confirmed by the project owner on 2026-11-09" + }, + "human_signoff": "attested", "limitations": [ - "This artifact is not a substitute for an external human review." + "Human approval applies only to the reviewed commit." ], "passed": true, "release_decision": { - "reason": "automated evidence is not a substitute for external human review", - "status": "block-vm-milestone" + "reason": "independent human review approved commit a111f923df43a66826b71ab70db8a484532fbf3b", + "status": "signoff-recorded" }, "review_type": "automated-independent-second-pass", "schema_version": 1 diff --git a/docs/observability.md b/docs/observability.md index 468678bd..ebf9b078 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -29,12 +29,23 @@ Business and domain modules do not select handlers or formats. The configured The mutation pipeline keeps its event construction in `r2morph.pipeline.observability`, separate from mutation policy. +Enhanced analysis emits a correlated start and completion event for each +`analyze` operation. Recoverable phase failures and unavailable optional phases +emit warning events while retaining the existing terminal guidance. + +CLI analysis, function listing, diff, validation, report, and cache operations +emit correlated lifecycle events. Validation also records its pass/fail result +separately from whether the validator itself ran successfully. + ## Configuration Explicit `setup_logging` arguments override environment values for the legacy `level` and `log_file` entry points. New applications should use `LoggingConfig` or the environment. +CLI commands preserve `R2MORPH_LOG_LEVEL` unless `--verbose` or `--debug` is +set; either flag forces `DEBUG` for that invocation. + | Variable | Default | Meaning | |---|---:|---| | `R2MORPH_LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR`, or `CRITICAL` | @@ -131,6 +142,9 @@ maps, or report payloads as log fields. Log counts, hashes, bounded previews, addresses, and typed outcomes instead. Sanitization is a final guardrail, not a reason to add sensitive inputs. +SARIF artifact URIs contain only input and output basenames. Reports omit the +host source root so sharing evidence does not disclose the analyst's paths. + ## Resilience and retention The CLI is low-volume and synchronous, so an asynchronous logging queue is not diff --git a/docs/one-zero-readiness.json b/docs/one-zero-readiness.json new file mode 100644 index 00000000..34eb3459 --- /dev/null +++ b/docs/one-zero-readiness.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "status": "blocked", + "blocking_criteria": [ + "all_named_adversarial_analyzers", + "complete_compatibility_contract", + "cross_platform_parity", + "published_correctness_results", + "published_cost_results", + "published_resistance_results", + "stable_api_cli" + ], + "criteria": { + "all_named_adversarial_analyzers": { + "status": "open", + "evidence": ["docs/compatibility-corpus.md", "docs/release-blockers.md"] + }, + "complete_compatibility_contract": { + "status": "in_progress", + "evidence": ["docs/support-matrix.json", "docs/compatibility-corpus.md"] + }, + "cross_platform_parity": { + "status": "open", + "evidence": ["docs/support-matrix.json", "docs/release-blockers.md"] + }, + "documented_resistance_scope": { + "status": "complete", + "evidence": ["README.md", "docs/independent-review-packet.md", "docs/protection-independent-analysis-2026-11-10.json"] + }, + "published_cost_results": { + "status": "in_progress", + "evidence": ["docs/pass-maturity.md", "docs/compatibility-corpus.md"] + }, + "published_correctness_results": { + "status": "in_progress", + "evidence": ["docs/compatibility-corpus.md", "docs/support-matrix.json"] + }, + "published_resistance_results": { + "status": "open", + "evidence": ["docs/protection-handler-clustering.json", "docs/protection-bytecode-grammar.json"] + }, + "stable_api_cli": { + "status": "in_progress", + "evidence": ["docs/stability-contract.md", "tests/unit/test_public_surface_contract.py"] + } + } +} diff --git a/docs/pass-maturity.md b/docs/pass-maturity.md index 01d72b13..3f69329e 100644 --- a/docs/pass-maturity.md +++ b/docs/pass-maturity.md @@ -7,6 +7,14 @@ promoted. Preview PE, Mach-O, ARM, and AArch64 evidence does not imply parity with the official Linux ELF x86-64 target. +`target_cells` is the canonical support contract for each pass. Each entry +names one exact format and architecture pair and its `support_status` +(`supported`, `preview`, or `experimental`). The legacy `formats` and +`architectures` lists are projections of those cells; they must not be +cross-multiplied to infer additional combinations. Evidence is tracked +separately: a declared target without evidence remains unevidenced, and a pair +without a `target_cells` entry makes no support claim. + The compatibility corpus selection currently covers `BlockReordering`, `CodeVirtualization`, `ConstantUnfolding`, `ControlFlowFlattening`, `DeadCodeInjection`, `InstructionExpansion`, `InstructionSubstitution`, @@ -31,7 +39,7 @@ contract until they have equivalent real-fixture coverage. | Instruction substitution | Tier 1 | Linux ELF x86-64; ELF ARM32, x86-32, AArch64, PE x86-64, and Mach-O AArch64/x86-64 preview | Rule coverage is ISA-specific; preview evidence is limited to focused real fixtures and does not imply format parity | `tests/product_smoke`, `tests/integration`, `tests/integration/test_mutation_nop_insertion_arm64.py`, `tests/integration/test_nop_insertion_pe_real.py`, `tests/integration/test_elf_arm32_native.py`, `tests/integration/test_elf_arm64_native.py`, `tests/integration/test_elf_x86_32_native.py`, `tests/integration/test_mutations_x86_compiled_binary.py` | | Register substitution | Tier 1 | Linux ELF x86-64; ELF ARM32, x86-32, AArch64, PE x86-64, and Mach-O AArch64/x86-64 preview | Requires proven liveness and ABI preservation; preview evidence is limited to focused real fixtures | `tests/product_smoke`, `tests/integration`, `tests/integration/test_nop_insertion_pe_real.py`, `tests/integration/test_mutation_register_substitution_arm64.py`, `tests/integration/test_elf_arm32_native.py`, `tests/integration/test_elf_arm64_native.py`, `tests/integration/test_elf_x86_32_native.py`, `tests/integration/test_mutations_x86_compiled_binary.py` | | Instruction expansion | Tier 1 | Linux ELF x86-64; ELF ARM32, x86-32, AArch64, PE x86-64, and Mach-O AArch64/x86-64 preview | Official only on Linux ELF x86-64; unsupported instruction families fail closed and preview evidence does not imply target parity | `tests/product_smoke/test_stable_mutations.py`, `tests/unit/test_instruction_expansion_helpers_contract.py`, `tests/integration/test_mutation_instruction_expansion_unsupported.py`, `docs/compatibility-corpus.md` | -| Block reordering | Tier 1 | ELF x86-64 | Eligible basic blocks are reordered with jump repair; complex indirect control flow is rejected before mutation | `tests/product_smoke/test_stable_mutations.py`, `tests/unit/test_block_reordering_edge_cases.py`, `tests/integration/test_block_reordering_apply_real.py`, `tests/integration/test_block_reordering_relocation_real.py`, `docs/compatibility-corpus.md` | +| Block reordering | Tier 1 | ELF x86-64; ELF x86-32, Mach-O x86-64, and PE x86-64 preview | Eligible basic blocks are reordered with jump repair; preview evidence uses an emulated ELF32 fixture and nine-input real PE and Mach-O fixtures; these cases do not establish broad format parity; complex indirect control flow is rejected before mutation | `tests/product_smoke/test_stable_mutations.py`, `tests/unit/test_block_reordering_edge_cases.py`, `tests/integration/test_block_reordering_apply_real.py`, `tests/integration/test_block_reordering_relocation_real.py`, `tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_block_reordering_preserves_emulated_exit_code`, `tests/integration/test_mutations_x86_compiled_binary.py::test_x86_block_reordering_preserves_native_exit_code`, `tests/integration/test_block_reordering_pe_real.py::test_pe_x86_64_block_reordering_preserves_native_observables`, `docs/compatibility-corpus.md` | | Dead code injection | Experimental | ELF x86-64 | Selected by the compatibility corpus and current composition smoke; placement depends on available safe regions and analyzer-effectiveness evidence remains incomplete | `tests/integration/test_dead_code_injection_flag_safe_real.py`, `tests/integration/test_protection_maturity_baseline.py`, `docs/compatibility-corpus.md` | | Control-flow flattening | Experimental | ELF x86-64 | Selected by the compatibility corpus and current composition smoke; cross-tool decompiler evidence remains incomplete | `tests/integration/test_control_flow_flattening_flag_safe_real.py`, `tests/integration/test_protection_maturity_baseline.py`, `docs/compatibility-corpus.md` | | Opaque predicates | Experimental | ELF x86-64 | Predicate families are not exhaustive | `README.md` | @@ -58,8 +66,8 @@ including official and non-official targets, so the Linux ELF x86-64 baseline and PE, Mach-O, ARM, and AArch64 parity gaps are visible without expanding every cell. It also records evidence percentages for official and non-official targets so preview coverage cannot look equivalent to the supported baseline. The -current matrix records 100.0% official evidence, 13.22% non-official evidence, -32 non-official evidenced cells, and 210 non-official not-supported cells. It +current matrix records 100.0% official evidence, 14.46% non-official evidence, +35 non-official evidenced cells, and 207 non-official not-supported cells. It also lists 11 non-official format/architecture targets in `non_official_gap_targets`; Mach-O AArch64, Mach-O x86-64, PE x86-64, and ELF ARM/AArch64/x86 currently have preview evidence, while Mach-O ARM/x86 and PE @@ -265,12 +273,16 @@ target handler stride diversity. These are resistance indicators, not human approval of anti-tamper or progressive bytecode protection. The generated artifacts also carry `adversarial_validation.status` as `pending-human-adversarial-review`, with automated adversarial smoke recorded as -the evidence quality. The scheduled adversarial workflow additionally runs six real +the evidence quality. This describes the automated artifacts; the separate +independent human approval is recorded in +[`human-review-signoff.json`](human-review-signoff.json). The scheduled adversarial workflow additionally runs six real tamper/progressive protection tests and publishes JUnit evidence, including single-level tamper, nested tamper, nested growth, and diversified-build -checks. This strengthens automated evidence but does not replace the pending -human/adversarial review. The generated `total_vm_resistance_blockers` count is -currently 1 VM resistance blockers (the remaining gap is external human review). +checks. The generated `total_vm_resistance_blockers` count is 0 VM resistance +blockers; independent review closed the human-resistance gap for the reviewed +commit. This zero covers only the declared campaign checks and review gate; it +does not establish general resistance to professional reverse engineering or +close the 1.0 `published_resistance_results` criterion. The complete local artifact [`protection-vm-resistance-2026-09-22-e3a491b3.json`](protection-vm-resistance-2026-09-22-e3a491b3.json) passes semantic parity, artifact, opcode, dispatcher, handler, grammar, @@ -287,4 +299,4 @@ ten-seed report that joins semantic parity, distinct artifact hashes, opcode-assignment, dispatcher/handler/stride diversity, single- and nested-layer tamper divergence, and progressive blob growth. Its `human_adversarial_review` field remains pending by contract, so the report is continuous automated evidence -rather than a human signoff. +rather than a human signoff. The independent human signoff is recorded separately. diff --git a/docs/protection-independent-analysis-2026-11-10.json b/docs/protection-independent-analysis-2026-11-10.json new file mode 100644 index 00000000..028748e7 --- /dev/null +++ b/docs/protection-independent-analysis-2026-11-10.json @@ -0,0 +1,65 @@ +{ + "schema_version": 1, + "measurement": "targeted-independent-static-analysis-pair", + "date_utc": "2026-11-10", + "source_commit": "1942151b", + "fixture": { + "path": "fixtures/dataset/elf_x86_64", + "sha256": "7b2dd51db978d08f362c2b7a0b6062521340207faf1955407361ae3a3268f50a", + "format": "ELF", + "architecture": "x86_64" + }, + "transformation": { + "pass": "CodeVirtualization", + "seed": 20261110, + "functions_virtualized": 1, + "instructions_virtualized": 9, + "bytecode_bytes": 9624, + "unsupported_functions": 0 + }, + "analyzer": { + "availability": "available", + "service": "connected licensed static-analysis service", + "decompiler_ready": true, + "version": "not exposed by the service" + }, + "original": { + "sha256": "7b2dd51db978d08f362c2b7a0b6062521340207faf1955407361ae3a3268f50a", + "function_count": 1, + "functions": [ + { + "name": "_start", + "size_bytes": 12, + "basic_blocks": 1, + "cyclomatic_complexity": 1, + "decompilation_summary": "direct system exit with status 0" + } + ] + }, + "protected": { + "sha256": "108e6f7e80e07c935eaf32392fa2e70793437354989e3a3cfcecdfd239713716", + "function_count": 2, + "functions": [ + { + "name": "j__start", + "size_bytes": 5 + }, + { + "name": "_start", + "size_bytes": 294, + "basic_blocks": 12, + "cyclomatic_complexity": 7, + "decompilation_summary": "byte loop with nested rotate and XOR operations" + } + ] + }, + "runtime_equivalence": "not measured", + "interpretation": "For this single ELF x86_64 fixture, the analyzed protected output differs structurally from the original decompilation. This is one bounded static-analysis observation.", + "limitations": [ + "one fixture and one pass only", + "analyzer version was not exposed", + "runtime equivalence was not executed or measured", + "does not establish general resistance to professional reverse engineering", + "does not replace the required three-analyzer release campaign" + ] +} diff --git a/docs/release-blockers.md b/docs/release-blockers.md index d9aee567..1309aa35 100644 --- a/docs/release-blockers.md +++ b/docs/release-blockers.md @@ -2,6 +2,20 @@ This ledger records the remaining blockers from the current release-hardening review. It is not a feature roadmap and must not be read as a support claim. +The machine-readable [1.0 readiness ledger](one-zero-readiness.json) records +the evidence requirements and keeps stable-major releases blocked while any +criterion remains open or in progress. + +## Latest full CI verification of a code-bearing commit + +Commit `4bd8aa85` passed CI run +[`42595479228`](https://github.com/seifreed/r2morph/actions/runs/42595479228) +with all 15 jobs successful, including the full Linux Python 3.12 suite, +integration tests, the Python/platform matrix, property and fuzz validation, +installed-wheel smoke tests, lint, and type checking. The Windows Python 3.12 +and 3.13 jobs both passed after adding bounded retries for transient radare2 +download failures. This result verifies that code commit only; later code +changes require their own successful run. ## Resolved blockers @@ -62,9 +76,12 @@ review. It is not a feature roadmap and must not be read as a support claim. non-official in the support matrix. - RB-005: The adversarial benchmark's required cross-tool campaign is resolved. Run `36381653200` contains complete comparable `radare2`, `angr`, and `ghidra` - rows for all 22 declared passes. Binary Ninja is deliberately on hold for - this milestone: it remains an explicit unavailable slot and is not silently - omitted, but no installation work is planned in the current campaign. + rows for all 22 declared passes. The Binary Ninja application bundle exists + in the local environment. Binary Ninja is deliberately on hold for this + milestone because the benchmark's real-fixture API probe returns + `unavailable` with `analyzer license is unavailable`. It remains an explicit + unavailable slot and is not counted as analyzer evidence. No licensed API is + available for this campaign. Evidence map: [compatibility-corpus.md](compatibility-corpus.md), [adversarial-benchmark.yml](../.github/workflows/adversarial-benchmark.yml) @@ -73,33 +90,39 @@ review. It is not a feature roadmap and must not be read as a support claim. Binary Ninja remains a recorded unavailable hold with its reason. The campaign is partitioned into deterministic fixture shards and merged before the corpus-wide gate runs; this addresses campaign sustainability. -- RB-006: The automated VM-resistance campaign is complete for fifteen fixtures - across ten seeds: semantic parity, opcode/handler/dispatcher diversity, - anti-tamper and progressive bytecode protection, plus bounded recovery - probes all passed. The remaining blocker is external human adversarial review; - automated evidence does not constitute human signoff. - Evidence map: [protection-vm-resistance-2026-09-22-e3a491b3.json](protection-vm-resistance-2026-09-22-e3a491b3.json), - [protection-vm-resistance-2026-09-20.json](protection-vm-resistance-2026-09-20.json), - [protection-handler-clustering.json](protection-handler-clustering.json), - [protection-bytecode-grammar.json](protection-bytecode-grammar.json) - `adversarial_validation` and `vm_resistance_blocker_totals`, plus the automated - tamper/progressive smoke in - [adversarial-benchmark.yml](../.github/workflows/adversarial-benchmark.yml). - Exit criteria: adversarial review validates VM diversity, anti-tamper, and - progressive bytecode protection instead of treating seed diversity or the - automated smoke as signoff. -- RB-007: The VM milestone remains blocked until external human review records signoff. - Evidence map: [independent-review.json](independent-review.json) - `human_signoff` and `release_decision`, - [independent-review-packet.md](independent-review-packet.md). Exit criteria: - external human review records signoff for the VM milestone and the release - decision no longer blocks the VM milestone. +## Resolved review gates + +- RB-006: The independent human review approved the VM resistance evidence on + commit `a111f923df43a66826b71ab70db8a484532fbf3b`. The reviewer inspected + VM diversification, dispatch and handler alternatives, anti-tamper and + progressive bytecode protection, fuzz properties, failure handling, and + bounded reporting. Evidence map: + [human-review-signoff.json](human-review-signoff.json), + [independent-review-packet.md](independent-review-packet.md), and + [independent-review.json](independent-review.json). The + [protection-handler-clustering.json](protection-handler-clustering.json) and + [protection-bytecode-grammar.json](protection-bytecode-grammar.json) + `adversarial_validation` fields retain the automated evidence scope; the + [support matrix](support-matrix.json) records zero `vm_resistance_blocker_totals`. + This resolves only the human signoff requirement. It does not establish + general resistance to professional reverse engineering or close the 1.0 + `published_resistance_results` criterion. + Exit criteria: the + approval remains bound to the reviewed commit; implementation changes require + another independent review. +- RB-007: Independent human review records signoff for the reviewed VM commit and + the release decision now reports `signoff-recorded`. The + `human_signoff` and `release_decision` fields record this state. Evidence map: + [human-review-signoff.json](human-review-signoff.json), + [independent-review.json](independent-review.json). Exit criteria: preserve + the review record and require a new review when the reviewed implementation + changes. The continuous fuzzing scope has an additional reproducible local run at [protection-fuzz-2026-09-22-d02bc5b2.json](protection-fuzz-2026-09-22-d02bc5b2.json): 20,000 target runs across 5,000 deterministic cases completed with zero -failures. This strengthens the fuzzing evidence but does not close the -remaining resistance and signoff blockers without adversarial and human review. +failures. This strengthens fuzzing evidence alongside the independent review; +it does not establish parity for preview or experimental targets. The current scheduled workflow also passed 20,000 target runs on commit `0933d50c`; its retained summary is [protection-fuzz-2026-09-23-0933d50c.json](protection-fuzz-2026-09-23-0933d50c.json). diff --git a/docs/stability-contract.md b/docs/stability-contract.md new file mode 100644 index 00000000..fd713229 --- /dev/null +++ b/docs/stability-contract.md @@ -0,0 +1,48 @@ +# Public API and CLI compatibility contract + +This contract defines the compatibility surface intended for the 1.0 release. +The current `0.4.0-alpha.3` release remains pre-stable; the contract does not +claim that 1.0 criteria have already been met. + +## Python API + +The package root is the supported import surface: + +```python +from r2morph import Binary, MorphEngine, Pipeline +``` + +`r2morph.__all__` and the behavior of those exported classes are covered by +`tests/unit/test_public_surface_contract.py`. CI also runs this contract +against the installed wheel on Linux and in the cross-platform package smoke +job. Modules not exported by the package root remain implementation details +unless a later version explicitly promotes them here. + +## CLI + +The installed `r2morph` command is bound to `r2morph.cli:app`. Its top-level +commands, the `morph`/`mutate` aliases, and the long options of `mutate` are +compatibility surfaces. The regression test for these names is +`tests/unit/test_public_surface_contract.py`, including when CI imports the +installed wheel rather than the source checkout. + +The default command, `analyze`, `functions`, `validate`, `diff`, `report`, +`version`, `cache`, and `experimental analyze-enhanced` are listed explicitly +so changes to command routing are reviewed as API changes. The short options +remain aliases for their corresponding long options. + +## Compatibility policy for 1.0 and later + +- A minor release may add Python exports, commands, options, and report fields. +- A minor release must not remove or change the meaning of a contracted export, + command, option, or report field. +- A removal or incompatible behavior change requires a deprecation period and a + major version change. The release notes must identify the replacement and + migration path. +- JSON and SARIF compatibility is governed by their checked-in schemas and + schema-version fields; schema changes require the associated contract tests. +- Experimental commands and mutation capabilities are excluded from stability + guarantees and must remain labeled experimental in help and documentation. + +Before 1.0, changes may still revise this contract. The 1.0 release gate must +require this document and its regression tests to match the shipped package. diff --git a/docs/support-matrix.json b/docs/support-matrix.json index 95eb9ac2..0491fd6f 100644 --- a/docs/support-matrix.json +++ b/docs/support-matrix.json @@ -24,10 +24,14 @@ "stability": "tier-1", "formats": [ "ELF", - "PE" + "PE", + "Mach-O" ], "architectures": [ - "x86-64" + "x86-64", + "x86", + "AArch64", + "ARM" ], "evidence_cells": [ { @@ -36,27 +40,45 @@ }, { "format": "ELF", - "architecture": "ARM" + "architecture": "ARM", + "evidence": [ + "tests/integration/test_elf_arm32_native.py::test_elf_arm32_nop_insertion_preserves_emulated_exit_code" + ] }, { "format": "ELF", - "architecture": "x86" + "architecture": "x86", + "evidence": [ + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_nop_insertion_preserves_native_exit_code" + ] }, { "format": "ELF", - "architecture": "AArch64" + "architecture": "AArch64", + "evidence": [ + "tests/integration/test_elf_arm64_native.py::test_elf_arm64_nop_insertion_preserves_native_exit_code" + ] }, { "format": "Mach-O", - "architecture": "AArch64" + "architecture": "AArch64", + "evidence": [ + "tests/integration/test_mutation_nop_insertion_arm64.py::test_nop_insertion_arm64_preserves_native_output" + ] }, { "format": "Mach-O", - "architecture": "x86-64" + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_nop_insertion_and_substitution_real" + ] }, { "format": "PE", - "architecture": "x86-64" + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_nop_insertion_pe_real.py::test_nop_insertion_pe_x86_64_preserves_repaired_integrity" + ] } ], "evidence": [ @@ -69,16 +91,58 @@ "tests/integration/test_elf_arm64_native.py", "tests/integration/test_mutations_x86_compiled_binary.py", "tests/integration/test_protection_maturity_baseline.py" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "supported" + }, + { + "format": "ELF", + "architecture": "ARM", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "x86", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "AArch64", + "support_status": "experimental" + }, + { + "format": "Mach-O", + "architecture": "AArch64", + "support_status": "experimental" + }, + { + "format": "Mach-O", + "architecture": "x86-64", + "support_status": "experimental" + }, + { + "format": "PE", + "architecture": "x86-64", + "support_status": "preview" + } ] }, { "name": "substitute", "stability": "tier-1", "formats": [ - "ELF" + "ELF", + "PE", + "Mach-O" ], "architectures": [ - "x86-64" + "x86-64", + "x86", + "AArch64", + "ARM" ], "evidence_cells": [ { @@ -87,27 +151,45 @@ }, { "format": "ELF", - "architecture": "ARM" + "architecture": "ARM", + "evidence": [ + "tests/integration/test_elf_arm32_native.py::test_elf_arm32_instruction_substitution_preserves_emulated_exit_code" + ] }, { "format": "ELF", - "architecture": "x86" + "architecture": "x86", + "evidence": [ + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_instruction_substitution_preserves_native_exit_code" + ] }, { "format": "ELF", - "architecture": "AArch64" + "architecture": "AArch64", + "evidence": [ + "tests/integration/test_elf_arm64_native.py::test_elf_arm64_instruction_substitution_preserves_native_exit_code" + ] }, { "format": "Mach-O", - "architecture": "AArch64" + "architecture": "AArch64", + "evidence": [ + "tests/integration/test_mutation_nop_insertion_arm64.py::test_instruction_substitution_arm64_preserves_native_output" + ] }, { "format": "Mach-O", - "architecture": "x86-64" + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_nop_insertion_and_substitution_real" + ] }, { "format": "PE", - "architecture": "x86-64" + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_nop_insertion_pe_real.py::test_instruction_substitution_pe_x86_64_preserves_native_execution" + ] } ], "evidence": [ @@ -118,16 +200,58 @@ "tests/integration/test_elf_x86_32_native.py", "tests/integration/test_elf_arm64_native.py", "tests/integration/test_mutation_passes_end_to_end_more2.py" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "supported" + }, + { + "format": "ELF", + "architecture": "ARM", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "x86", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "AArch64", + "support_status": "experimental" + }, + { + "format": "Mach-O", + "architecture": "AArch64", + "support_status": "experimental" + }, + { + "format": "Mach-O", + "architecture": "x86-64", + "support_status": "experimental" + }, + { + "format": "PE", + "architecture": "x86-64", + "support_status": "preview" + } ] }, { "name": "register", "stability": "tier-1", "formats": [ - "ELF" + "ELF", + "PE", + "Mach-O" ], "architectures": [ - "x86-64" + "x86-64", + "x86", + "AArch64", + "ARM" ], "evidence_cells": [ { @@ -136,27 +260,45 @@ }, { "format": "ELF", - "architecture": "ARM" + "architecture": "ARM", + "evidence": [ + "tests/integration/test_elf_arm32_native.py::test_elf_arm32_register_substitution_preserves_emulated_exit_code" + ] }, { "format": "ELF", - "architecture": "x86" + "architecture": "x86", + "evidence": [ + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_register_substitution_preserves_native_exit_code" + ] }, { "format": "ELF", - "architecture": "AArch64" + "architecture": "AArch64", + "evidence": [ + "tests/integration/test_elf_arm64_native.py::test_elf_arm64_register_substitution_preserves_native_exit_code" + ] }, { "format": "Mach-O", - "architecture": "AArch64" + "architecture": "AArch64", + "evidence": [ + "tests/integration/test_mutation_register_substitution_arm64.py::test_register_substitution_arm64_preserves_generated_native_execution" + ] }, { "format": "Mach-O", - "architecture": "x86-64" + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_instruction_expansion_and_register_substitution_real" + ] }, { "format": "PE", - "architecture": "x86-64" + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_nop_insertion_pe_real.py::test_register_substitution_pe_x86_64_preserves_native_execution" + ] } ], "evidence": [ @@ -167,16 +309,58 @@ "tests/integration/test_elf_x86_32_native.py", "tests/integration/test_elf_arm64_native.py", "tests/integration/test_nop_insertion_pe_real.py" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "supported" + }, + { + "format": "ELF", + "architecture": "ARM", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "x86", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "AArch64", + "support_status": "experimental" + }, + { + "format": "Mach-O", + "architecture": "AArch64", + "support_status": "experimental" + }, + { + "format": "Mach-O", + "architecture": "x86-64", + "support_status": "experimental" + }, + { + "format": "PE", + "architecture": "x86-64", + "support_status": "preview" + } ] }, { "name": "instruction-expansion", "stability": "tier-1", "formats": [ - "ELF" + "ELF", + "PE", + "Mach-O" ], "architectures": [ - "x86-64" + "x86-64", + "x86", + "AArch64", + "ARM" ], "evidence_cells": [ { @@ -185,27 +369,45 @@ }, { "format": "ELF", - "architecture": "AArch64" + "architecture": "AArch64", + "evidence": [ + "tests/integration/test_elf_arm64_native.py::test_elf_arm64_instruction_expansion_preserves_native_exit_code" + ] }, { "format": "ELF", - "architecture": "ARM" + "architecture": "ARM", + "evidence": [ + "tests/integration/test_elf_arm32_native.py::test_elf_arm32_instruction_expansion_preserves_emulated_exit_code" + ] }, { "format": "ELF", - "architecture": "x86" + "architecture": "x86", + "evidence": [ + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_instruction_expansion_preserves_native_exit_code" + ] }, { "format": "PE", - "architecture": "x86-64" + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_nop_insertion_pe_real.py::test_instruction_expansion_pe_x86_64_preserves_native_execution" + ] }, { "format": "Mach-O", - "architecture": "AArch64" + "architecture": "AArch64", + "evidence": [ + "tests/integration/test_mutation_nop_insertion_arm64.py::test_instruction_expansion_arm64_preserves_native_output" + ] }, { "format": "Mach-O", - "architecture": "x86-64" + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_instruction_expansion_and_register_substitution_real" + ] } ], "evidence": [ @@ -213,29 +415,113 @@ "tests/unit/test_instruction_expansion_helpers_contract.py", "tests/integration/test_mutation_instruction_expansion_unsupported.py", "docs/compatibility-corpus.md" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "supported" + }, + { + "format": "ELF", + "architecture": "AArch64", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "ARM", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "x86", + "support_status": "experimental" + }, + { + "format": "PE", + "architecture": "x86-64", + "support_status": "preview" + }, + { + "format": "Mach-O", + "architecture": "AArch64", + "support_status": "experimental" + }, + { + "format": "Mach-O", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { "name": "block-reordering", "stability": "tier-1", "formats": [ - "ELF" + "ELF", + "PE", + "Mach-O" ], "architectures": [ - "x86-64" + "x86-64", + "x86" ], "evidence_cells": [ { - "format": "ELF", - "architecture": "x86-64" + "format": "ELF", + "architecture": "x86-64" + }, + { + "format": "ELF", + "architecture": "x86", + "evidence": [ + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_block_reordering_preserves_emulated_exit_code" + ] + }, + { + "format": "Mach-O", + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_block_reordering_preserves_native_exit_code" + ] + }, + { + "format": "PE", + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_block_reordering_pe_real.py::test_pe_x86_64_block_reordering_preserves_native_observables" + ] + } + ], + "evidence": [ + "tests/product_smoke/test_stable_mutations.py", + "tests/unit/test_block_reordering_edge_cases.py", + "tests/integration/test_block_reordering_apply_real.py", + "tests/integration/test_block_reordering_relocation_real.py", + "tests/integration/test_block_reordering_pe_real.py", + "docs/compatibility-corpus.md" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "supported" + }, + { + "format": "ELF", + "architecture": "x86", + "support_status": "experimental" + }, + { + "format": "Mach-O", + "architecture": "x86-64", + "support_status": "experimental" + }, + { + "format": "PE", + "architecture": "x86-64", + "support_status": "preview" } - ], - "evidence": [ - "tests/product_smoke/test_stable_mutations.py", - "tests/unit/test_block_reordering_edge_cases.py", - "tests/integration/test_block_reordering_apply_real.py", - "tests/integration/test_block_reordering_relocation_real.py", - "docs/compatibility-corpus.md" ] }, { @@ -256,6 +542,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -276,6 +569,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -296,6 +596,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -317,6 +624,13 @@ "docs/protection-maturity.md", "tests/integration", "tests/integration/test_code_virtualization_generic_isa_real.py" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -337,6 +651,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -357,6 +678,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -377,16 +705,28 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { "name": "constant-unfolding", "stability": "experimental", "formats": [ - "ELF" + "ELF", + "PE", + "Mach-O" ], "architectures": [ - "x86-64" + "x86-64", + "x86", + "AArch64", + "ARM" ], "evidence_cells": [ { @@ -395,23 +735,38 @@ }, { "format": "ELF", - "architecture": "ARM" + "architecture": "ARM", + "evidence": [ + "tests/integration/test_elf_arm32_native.py::test_elf_arm32_constant_unfolding_zero_preserves_emulated_exit_code" + ] }, { "format": "ELF", - "architecture": "AArch64" + "architecture": "AArch64", + "evidence": [ + "tests/integration/test_elf_arm64_native.py::test_elf_arm64_constant_unfolding_zero_preserves_native_exit_code" + ] }, { "format": "ELF", - "architecture": "x86" + "architecture": "x86", + "evidence": [ + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_constant_unfolding_zero_preserves_native_exit_code" + ] }, { "format": "Mach-O", - "architecture": "AArch64" + "architecture": "AArch64", + "evidence": [ + "tests/integration/test_mutation_nop_insertion_arm64.py::test_constant_unfolding_arm64_preserves_native_output" + ] }, { "format": "PE", - "architecture": "x86-64" + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_nop_insertion_pe_real.py::test_constant_unfolding_pe_x86_64_preserves_native_execution" + ] } ], "evidence": [ @@ -422,6 +777,38 @@ "tests/integration/test_elf_x86_32_native.py", "tests/integration/test_nop_insertion_pe_real.py", "tests/integration/test_mutation_nop_insertion_arm64.py" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "ARM", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "AArch64", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "x86", + "support_status": "experimental" + }, + { + "format": "Mach-O", + "architecture": "AArch64", + "support_status": "experimental" + }, + { + "format": "PE", + "architecture": "x86-64", + "support_status": "preview" + } ] }, { @@ -442,6 +829,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -462,6 +856,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -482,6 +883,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -502,6 +910,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -522,6 +937,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -542,16 +964,26 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { "name": "string-obfuscation", "stability": "experimental", "formats": [ - "ELF" + "ELF", + "PE", + "Mach-O" ], "architectures": [ - "x86-64" + "x86-64", + "x86" ], "evidence_cells": [ { @@ -560,15 +992,24 @@ }, { "format": "ELF", - "architecture": "x86" + "architecture": "x86", + "evidence": [ + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_string_obfuscation_preserves_native_exit_code" + ] }, { "format": "Mach-O", - "architecture": "x86-64" + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_string_obfuscation_preserves_native_exit_code" + ] }, { "format": "PE", - "architecture": "x86-64" + "architecture": "x86-64", + "evidence": [ + "tests/integration/test_nop_insertion_pe_real.py::test_string_obfuscation_pe_x86_64_preserves_native_execution" + ] } ], "evidence": [ @@ -576,6 +1017,28 @@ "tests/integration/test_elf_x86_32_native.py", "tests/integration/test_mutations_x86_compiled_binary.py", "tests/integration/test_nop_insertion_pe_real.py" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + }, + { + "format": "ELF", + "architecture": "x86", + "support_status": "experimental" + }, + { + "format": "Mach-O", + "architecture": "x86-64", + "support_status": "experimental" + }, + { + "format": "PE", + "architecture": "x86-64", + "support_status": "preview" + } ] }, { @@ -596,6 +1059,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] }, { @@ -616,6 +1086,13 @@ "evidence": [ "README.md", "tests/integration" + ], + "target_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "support_status": "experimental" + } ] } ], @@ -1075,14 +1552,21 @@ }, "cell_count": 264, "summary": { - "evidenced_cells": 54, - "not_supported_cells": 210, + "declared_support_cells": 57, + "unclaimed_cells": 207, + "target_support_status_counts": { + "experimental": 45, + "preview": 7, + "supported": 5 + }, + "evidenced_cells": 57, + "not_supported_cells": 207, "official_evidenced_cells": 22, "official_not_supported_cells": 0, "official_evidence_percent": 100.0, - "non_official_evidenced_cells": 32, - "non_official_not_supported_cells": 210, - "non_official_evidence_percent": 13.22, + "non_official_evidenced_cells": 35, + "non_official_not_supported_cells": 207, + "non_official_evidence_percent": 14.46, "non_official_gap_targets": [ { "format": "ELF", @@ -1101,9 +1585,9 @@ { "format": "ELF", "architecture": "x86", - "evidenced_cells": 6, - "not_supported_cells": 16, - "evidence_percent": 27.27 + "evidenced_cells": 7, + "not_supported_cells": 15, + "evidence_percent": 31.82 }, { "format": "Mach-O", @@ -1129,9 +1613,9 @@ { "format": "Mach-O", "architecture": "x86-64", - "evidenced_cells": 5, - "not_supported_cells": 17, - "evidence_percent": 22.73 + "evidenced_cells": 6, + "not_supported_cells": 16, + "evidence_percent": 27.27 }, { "format": "PE", @@ -1157,9 +1641,9 @@ { "format": "PE", "architecture": "x86-64", - "evidenced_cells": 6, - "not_supported_cells": 16, - "evidence_percent": 27.27 + "evidenced_cells": 7, + "not_supported_cells": 15, + "evidence_percent": 31.82 } ], "parity_gap_scope": { @@ -1192,9 +1676,9 @@ { "format": "ELF", "architecture": "x86", - "evidenced_cells": 6, - "not_supported_cells": 16, - "evidence_percent": 27.27 + "evidenced_cells": 7, + "not_supported_cells": 15, + "evidence_percent": 31.82 }, { "format": "Mach-O", @@ -1206,16 +1690,16 @@ { "format": "Mach-O", "architecture": "x86-64", - "evidenced_cells": 5, - "not_supported_cells": 17, - "evidence_percent": 22.73 + "evidenced_cells": 6, + "not_supported_cells": 16, + "evidence_percent": 27.27 }, { "format": "PE", "architecture": "x86-64", - "evidenced_cells": 6, - "not_supported_cells": 16, - "evidence_percent": 27.27 + "evidenced_cells": 7, + "not_supported_cells": 15, + "evidence_percent": 31.82 } ], "zero_evidence_targets": [ @@ -1275,9 +1759,9 @@ { "format": "ELF", "architecture": "x86", - "evidenced_cells": 6, - "not_supported_cells": 16, - "evidence_percent": 27.27 + "evidenced_cells": 7, + "not_supported_cells": 15, + "evidence_percent": 31.82 }, { "format": "Mach-O", @@ -1303,9 +1787,9 @@ { "format": "Mach-O", "architecture": "x86-64", - "evidenced_cells": 5, - "not_supported_cells": 17, - "evidence_percent": 22.73 + "evidenced_cells": 6, + "not_supported_cells": 16, + "evidence_percent": 27.27 }, { "format": "PE", @@ -1331,9 +1815,9 @@ { "format": "PE", "architecture": "x86-64", - "evidenced_cells": 6, - "not_supported_cells": 16, - "evidence_percent": 27.27 + "evidenced_cells": 7, + "not_supported_cells": 15, + "evidence_percent": 31.82 } ], "parity_gap_scope": { @@ -1349,7 +1833,7 @@ } }, "parity_blocker_totals": { - "non_official_missing_evidence_cells": 210, + "non_official_missing_evidence_cells": 207, "non_official_gap_targets": 11, "parity_gap_architectures": 3, "parity_gap_formats": 2, @@ -1789,36 +2273,29 @@ "total_vm_semantic_blockers": 0, "vm_semantic_gap_scope": 0 }, - "vm_resistance_gap_scope": [ - "human-adversarial-validation" - ], - "vm_resistance_gap_evidence": { + "vm_resistance_gap_scope": [], + "vm_resistance_gap_evidence": {}, + "vm_resistance_resolved_evidence": { "human-adversarial-validation": { - "status": "pending-human-adversarial-review", - "evidence_quality": "automated-adversarial-smoke", + "status": "approved", + "evidence_quality": "independent-human-review", + "reviewed_commit": "a111f923df43a66826b71ab70db8a484532fbf3b", + "reviewer": "Marc Rivero Lopez", + "review_date_utc": "2026-11-08", + "specialist_expertise_confirmation": "binary rewriting and virtualization experience confirmed by the project owner on 2026-11-09", "evidence": [ - "docs/independent-review.json", - "docs/protection-bytecode-grammar.json", - "docs/protection-handler-clustering.json", - "docs/protection-vm-resistance-2026-09-22-e3a491b3.json", - "docs/protection-vm-resistance-2026-09-19-3feec419.json", - "docs/protection-vm-resistance-2026-09-20.json", - ".github/workflows/adversarial-benchmark.yml", - "scripts/vm_resistance_adversarial.py", - "tests/integration/test_code_virtualization_real.py", - "tests/integration/test_code_virtualization_diversification_real.py", - "tests/integration/test_vm_resistance_adversarial.py" + "docs/human-review-signoff.json", + "docs/independent-review-packet.md", + "docs/protection-vm-resistance-2026-09-22-e3a491b3.json" ] } }, "vm_resistance_evidence_blockers": { - "vm_resistance_gap_scope": [ - "human-adversarial-validation" - ] + "vm_resistance_gap_scope": [] }, "vm_resistance_blocker_totals": { - "total_vm_resistance_blockers": 1, - "vm_resistance_gap_scope": 1 + "total_vm_resistance_blockers": 0, + "vm_resistance_gap_scope": 0 } }, "cells": [ @@ -1827,6 +2304,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "supported", "status": "evidenced", "evidence": [ "tests/product_smoke", @@ -1844,72 +2322,44 @@ "pass": "nop", "format": "ELF", "architecture": "x86", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_nop_insertion_pe_real.py", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutations_x86_compiled_binary.py", - "tests/integration/test_protection_maturity_baseline.py" + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_nop_insertion_preserves_native_exit_code" ] }, { "pass": "nop", "format": "ELF", "architecture": "AArch64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_nop_insertion_pe_real.py", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutations_x86_compiled_binary.py", - "tests/integration/test_protection_maturity_baseline.py" + "tests/integration/test_elf_arm64_native.py::test_elf_arm64_nop_insertion_preserves_native_exit_code" ] }, { "pass": "nop", "format": "ELF", "architecture": "ARM", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_nop_insertion_pe_real.py", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutations_x86_compiled_binary.py", - "tests/integration/test_protection_maturity_baseline.py" + "tests/integration/test_elf_arm32_native.py::test_elf_arm32_nop_insertion_preserves_emulated_exit_code" ] }, { - "pass": "nop", - "format": "PE", - "architecture": "x86-64", - "declared_support": true, - "status": "evidenced", - "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_nop_insertion_pe_real.py", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutations_x86_compiled_binary.py", - "tests/integration/test_protection_maturity_baseline.py" + "pass": "nop", + "format": "PE", + "architecture": "x86-64", + "declared_support": true, + "support_status": "preview", + "status": "evidenced", + "evidence": [ + "tests/integration/test_nop_insertion_pe_real.py::test_nop_insertion_pe_x86_64_preserves_repaired_integrity" ] }, { @@ -1917,6 +2367,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -1925,6 +2376,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -1933,6 +2385,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -1940,18 +2393,11 @@ "pass": "nop", "format": "Mach-O", "architecture": "x86-64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_nop_insertion_pe_real.py", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutations_x86_compiled_binary.py", - "tests/integration/test_protection_maturity_baseline.py" + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_nop_insertion_and_substitution_real" ] }, { @@ -1959,6 +2405,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -1966,18 +2413,11 @@ "pass": "nop", "format": "Mach-O", "architecture": "AArch64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_nop_insertion_pe_real.py", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutations_x86_compiled_binary.py", - "tests/integration/test_protection_maturity_baseline.py" + "tests/integration/test_mutation_nop_insertion_arm64.py::test_nop_insertion_arm64_preserves_native_output" ] }, { @@ -1985,6 +2425,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -1993,6 +2434,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "supported", "status": "evidenced", "evidence": [ "tests/product_smoke", @@ -2008,64 +2450,44 @@ "pass": "substitute", "format": "ELF", "architecture": "x86", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutation_passes_end_to_end_more2.py" + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_instruction_substitution_preserves_native_exit_code" ] }, { "pass": "substitute", "format": "ELF", "architecture": "AArch64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutation_passes_end_to_end_more2.py" + "tests/integration/test_elf_arm64_native.py::test_elf_arm64_instruction_substitution_preserves_native_exit_code" ] }, { "pass": "substitute", "format": "ELF", "architecture": "ARM", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutation_passes_end_to_end_more2.py" + "tests/integration/test_elf_arm32_native.py::test_elf_arm32_instruction_substitution_preserves_emulated_exit_code" ] }, { "pass": "substitute", "format": "PE", "architecture": "x86-64", - "declared_support": false, + "declared_support": true, + "support_status": "preview", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutation_passes_end_to_end_more2.py" + "tests/integration/test_nop_insertion_pe_real.py::test_instruction_substitution_pe_x86_64_preserves_native_execution" ] }, { @@ -2073,6 +2495,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2081,6 +2504,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2089,6 +2513,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2096,16 +2521,11 @@ "pass": "substitute", "format": "Mach-O", "architecture": "x86-64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutation_passes_end_to_end_more2.py" + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_nop_insertion_and_substitution_real" ] }, { @@ -2113,6 +2533,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2120,16 +2541,11 @@ "pass": "substitute", "format": "Mach-O", "architecture": "AArch64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_nop_insertion_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_mutation_passes_end_to_end_more2.py" + "tests/integration/test_mutation_nop_insertion_arm64.py::test_instruction_substitution_arm64_preserves_native_output" ] }, { @@ -2137,6 +2553,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2145,6 +2562,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "supported", "status": "evidenced", "evidence": [ "tests/product_smoke", @@ -2160,64 +2578,44 @@ "pass": "register", "format": "ELF", "architecture": "x86", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_register_substitution_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_nop_insertion_pe_real.py" + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_register_substitution_preserves_native_exit_code" ] }, { "pass": "register", "format": "ELF", "architecture": "AArch64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_register_substitution_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_nop_insertion_pe_real.py" + "tests/integration/test_elf_arm64_native.py::test_elf_arm64_register_substitution_preserves_native_exit_code" ] }, { "pass": "register", "format": "ELF", "architecture": "ARM", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_register_substitution_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_nop_insertion_pe_real.py" + "tests/integration/test_elf_arm32_native.py::test_elf_arm32_register_substitution_preserves_emulated_exit_code" ] }, { "pass": "register", "format": "PE", "architecture": "x86-64", - "declared_support": false, + "declared_support": true, + "support_status": "preview", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_register_substitution_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_nop_insertion_pe_real.py" + "tests/integration/test_nop_insertion_pe_real.py::test_register_substitution_pe_x86_64_preserves_native_execution" ] }, { @@ -2225,6 +2623,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2233,6 +2632,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2241,6 +2641,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2248,16 +2649,11 @@ "pass": "register", "format": "Mach-O", "architecture": "x86-64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_register_substitution_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_nop_insertion_pe_real.py" + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_instruction_expansion_and_register_substitution_real" ] }, { @@ -2265,6 +2661,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2272,16 +2669,11 @@ "pass": "register", "format": "Mach-O", "architecture": "AArch64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke", - "tests/integration", - "tests/integration/test_mutation_register_substitution_arm64.py", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_nop_insertion_pe_real.py" + "tests/integration/test_mutation_register_substitution_arm64.py::test_register_substitution_arm64_preserves_generated_native_execution" ] }, { @@ -2289,6 +2681,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2297,6 +2690,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "supported", "status": "evidenced", "evidence": [ "tests/product_smoke/test_stable_mutations.py", @@ -2309,52 +2703,44 @@ "pass": "instruction-expansion", "format": "ELF", "architecture": "x86", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke/test_stable_mutations.py", - "tests/unit/test_instruction_expansion_helpers_contract.py", - "tests/integration/test_mutation_instruction_expansion_unsupported.py", - "docs/compatibility-corpus.md" + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_instruction_expansion_preserves_native_exit_code" ] }, { "pass": "instruction-expansion", "format": "ELF", "architecture": "AArch64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke/test_stable_mutations.py", - "tests/unit/test_instruction_expansion_helpers_contract.py", - "tests/integration/test_mutation_instruction_expansion_unsupported.py", - "docs/compatibility-corpus.md" + "tests/integration/test_elf_arm64_native.py::test_elf_arm64_instruction_expansion_preserves_native_exit_code" ] }, { "pass": "instruction-expansion", "format": "ELF", "architecture": "ARM", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke/test_stable_mutations.py", - "tests/unit/test_instruction_expansion_helpers_contract.py", - "tests/integration/test_mutation_instruction_expansion_unsupported.py", - "docs/compatibility-corpus.md" + "tests/integration/test_elf_arm32_native.py::test_elf_arm32_instruction_expansion_preserves_emulated_exit_code" ] }, { "pass": "instruction-expansion", "format": "PE", "architecture": "x86-64", - "declared_support": false, + "declared_support": true, + "support_status": "preview", "status": "evidenced", "evidence": [ - "tests/product_smoke/test_stable_mutations.py", - "tests/unit/test_instruction_expansion_helpers_contract.py", - "tests/integration/test_mutation_instruction_expansion_unsupported.py", - "docs/compatibility-corpus.md" + "tests/integration/test_nop_insertion_pe_real.py::test_instruction_expansion_pe_x86_64_preserves_native_execution" ] }, { @@ -2362,6 +2748,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2370,6 +2757,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2378,6 +2766,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2385,13 +2774,11 @@ "pass": "instruction-expansion", "format": "Mach-O", "architecture": "x86-64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke/test_stable_mutations.py", - "tests/unit/test_instruction_expansion_helpers_contract.py", - "tests/integration/test_mutation_instruction_expansion_unsupported.py", - "docs/compatibility-corpus.md" + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_instruction_expansion_and_register_substitution_real" ] }, { @@ -2399,6 +2786,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2406,13 +2794,11 @@ "pass": "instruction-expansion", "format": "Mach-O", "architecture": "AArch64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "tests/product_smoke/test_stable_mutations.py", - "tests/unit/test_instruction_expansion_helpers_contract.py", - "tests/integration/test_mutation_instruction_expansion_unsupported.py", - "docs/compatibility-corpus.md" + "tests/integration/test_mutation_nop_insertion_arm64.py::test_instruction_expansion_arm64_preserves_native_output" ] }, { @@ -2420,6 +2806,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2428,12 +2815,14 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "supported", "status": "evidenced", "evidence": [ "tests/product_smoke/test_stable_mutations.py", "tests/unit/test_block_reordering_edge_cases.py", "tests/integration/test_block_reordering_apply_real.py", "tests/integration/test_block_reordering_relocation_real.py", + "tests/integration/test_block_reordering_pe_real.py", "docs/compatibility-corpus.md" ] }, @@ -2441,15 +2830,19 @@ "pass": "block-reordering", "format": "ELF", "architecture": "x86", - "declared_support": false, - "status": "not-supported", - "evidence": [] + "declared_support": true, + "support_status": "experimental", + "status": "evidenced", + "evidence": [ + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_block_reordering_preserves_emulated_exit_code" + ] }, { "pass": "block-reordering", "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2458,6 +2851,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2465,15 +2859,19 @@ "pass": "block-reordering", "format": "PE", "architecture": "x86-64", - "declared_support": false, - "status": "not-supported", - "evidence": [] + "declared_support": true, + "support_status": "preview", + "status": "evidenced", + "evidence": [ + "tests/integration/test_block_reordering_pe_real.py::test_pe_x86_64_block_reordering_preserves_native_observables" + ] }, { "pass": "block-reordering", "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2482,6 +2880,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2490,6 +2889,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2497,15 +2897,19 @@ "pass": "block-reordering", "format": "Mach-O", "architecture": "x86-64", - "declared_support": false, - "status": "not-supported", - "evidence": [] + "declared_support": true, + "support_status": "experimental", + "status": "evidenced", + "evidence": [ + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_block_reordering_preserves_native_exit_code" + ] }, { "pass": "block-reordering", "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2514,6 +2918,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2522,6 +2927,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2530,6 +2936,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -2541,6 +2948,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2549,6 +2957,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2557,6 +2966,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2565,6 +2975,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2573,6 +2984,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2581,6 +2993,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2589,6 +3002,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2597,6 +3011,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2605,6 +3020,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2613,6 +3029,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2621,6 +3038,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2629,6 +3047,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -2640,6 +3059,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2648,6 +3068,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2656,6 +3077,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2664,6 +3086,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2672,6 +3095,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2680,6 +3104,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2688,6 +3113,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2696,6 +3122,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2704,6 +3131,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2712,6 +3140,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2720,6 +3149,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2728,6 +3158,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -2739,6 +3170,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2747,6 +3179,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2755,6 +3188,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2763,6 +3197,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2771,6 +3206,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2779,6 +3215,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2787,6 +3224,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2795,6 +3233,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2803,6 +3242,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2811,6 +3251,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2819,6 +3260,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2827,6 +3269,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "docs/protection-maturity.md", @@ -2839,6 +3282,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2847,6 +3291,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2855,6 +3300,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2863,6 +3309,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2871,6 +3318,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2879,6 +3327,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2887,6 +3336,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2895,6 +3345,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2903,6 +3354,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2911,6 +3363,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2919,6 +3372,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2927,6 +3381,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -2938,6 +3393,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2946,6 +3402,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2954,6 +3411,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2962,6 +3420,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2970,6 +3429,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2978,6 +3438,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2986,6 +3447,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -2994,6 +3456,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3002,6 +3465,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3010,6 +3474,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3018,6 +3483,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3026,6 +3492,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -3037,6 +3504,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3045,6 +3513,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3053,6 +3522,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3061,6 +3531,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3069,6 +3540,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3077,6 +3549,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3085,6 +3558,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3093,6 +3567,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3101,6 +3576,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3109,6 +3585,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3117,6 +3594,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3125,6 +3603,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -3136,6 +3615,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3144,6 +3624,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3152,6 +3633,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3160,6 +3642,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3168,6 +3651,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3176,6 +3660,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3184,6 +3669,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3192,6 +3678,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3200,6 +3687,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3208,6 +3696,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3216,6 +3705,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3224,6 +3714,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -3239,64 +3730,44 @@ "pass": "constant-unfolding", "format": "ELF", "architecture": "x86", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "README.md", - "tests/integration", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_nop_insertion_pe_real.py", - "tests/integration/test_mutation_nop_insertion_arm64.py" + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_constant_unfolding_zero_preserves_native_exit_code" ] }, { "pass": "constant-unfolding", "format": "ELF", "architecture": "AArch64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "README.md", - "tests/integration", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_nop_insertion_pe_real.py", - "tests/integration/test_mutation_nop_insertion_arm64.py" + "tests/integration/test_elf_arm64_native.py::test_elf_arm64_constant_unfolding_zero_preserves_native_exit_code" ] }, { "pass": "constant-unfolding", "format": "ELF", "architecture": "ARM", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "README.md", - "tests/integration", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_nop_insertion_pe_real.py", - "tests/integration/test_mutation_nop_insertion_arm64.py" + "tests/integration/test_elf_arm32_native.py::test_elf_arm32_constant_unfolding_zero_preserves_emulated_exit_code" ] }, { "pass": "constant-unfolding", "format": "PE", "architecture": "x86-64", - "declared_support": false, + "declared_support": true, + "support_status": "preview", "status": "evidenced", "evidence": [ - "README.md", - "tests/integration", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_nop_insertion_pe_real.py", - "tests/integration/test_mutation_nop_insertion_arm64.py" + "tests/integration/test_nop_insertion_pe_real.py::test_constant_unfolding_pe_x86_64_preserves_native_execution" ] }, { @@ -3304,6 +3775,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3312,6 +3784,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3320,6 +3793,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3328,6 +3802,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3336,6 +3811,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3343,16 +3819,11 @@ "pass": "constant-unfolding", "format": "Mach-O", "architecture": "AArch64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "README.md", - "tests/integration", - "tests/integration/test_elf_arm32_native.py", - "tests/integration/test_elf_arm64_native.py", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_nop_insertion_pe_real.py", - "tests/integration/test_mutation_nop_insertion_arm64.py" + "tests/integration/test_mutation_nop_insertion_arm64.py::test_constant_unfolding_arm64_preserves_native_output" ] }, { @@ -3360,6 +3831,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3368,6 +3840,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -3379,6 +3852,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3387,6 +3861,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3395,6 +3870,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3403,6 +3879,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3411,6 +3888,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3419,6 +3897,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3427,6 +3906,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3435,6 +3915,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3443,6 +3924,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3451,6 +3933,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3459,6 +3942,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3467,6 +3951,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -3478,6 +3963,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3486,6 +3972,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3494,6 +3981,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3502,6 +3990,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3510,6 +3999,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3518,6 +4008,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3526,6 +4017,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3534,6 +4026,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3542,6 +4035,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3550,6 +4044,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3558,6 +4053,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3566,6 +4062,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -3577,6 +4074,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3585,6 +4083,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3593,6 +4092,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3601,6 +4101,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3609,6 +4110,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3617,6 +4119,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3625,6 +4128,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3633,6 +4137,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3641,6 +4146,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3649,6 +4155,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3657,6 +4164,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3665,6 +4173,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -3676,6 +4185,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3684,6 +4194,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3692,6 +4203,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3700,6 +4212,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3708,6 +4221,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3716,6 +4230,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3724,6 +4239,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3732,6 +4248,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3740,6 +4257,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3748,6 +4266,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3756,6 +4275,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3764,6 +4284,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -3775,6 +4296,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3783,6 +4305,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3791,6 +4314,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3799,6 +4323,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3807,6 +4332,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3815,6 +4341,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3823,6 +4350,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3831,6 +4359,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3839,6 +4368,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3847,6 +4377,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3855,6 +4386,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3863,6 +4395,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -3874,6 +4407,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3882,6 +4416,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3890,6 +4425,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3898,6 +4434,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3906,6 +4443,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3914,6 +4452,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3922,6 +4461,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3930,6 +4470,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3938,6 +4479,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3946,6 +4488,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3954,6 +4497,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3962,6 +4506,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -3974,13 +4519,11 @@ "pass": "string-obfuscation", "format": "ELF", "architecture": "x86", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "README.md", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_mutations_x86_compiled_binary.py", - "tests/integration/test_nop_insertion_pe_real.py" + "tests/integration/test_elf_x86_32_native.py::test_elf_x86_32_string_obfuscation_preserves_native_exit_code" ] }, { @@ -3988,6 +4531,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -3996,6 +4540,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4003,13 +4548,11 @@ "pass": "string-obfuscation", "format": "PE", "architecture": "x86-64", - "declared_support": false, + "declared_support": true, + "support_status": "preview", "status": "evidenced", "evidence": [ - "README.md", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_mutations_x86_compiled_binary.py", - "tests/integration/test_nop_insertion_pe_real.py" + "tests/integration/test_nop_insertion_pe_real.py::test_string_obfuscation_pe_x86_64_preserves_native_execution" ] }, { @@ -4017,6 +4560,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4025,6 +4569,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4033,6 +4578,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4040,13 +4586,11 @@ "pass": "string-obfuscation", "format": "Mach-O", "architecture": "x86-64", - "declared_support": false, + "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ - "README.md", - "tests/integration/test_elf_x86_32_native.py", - "tests/integration/test_mutations_x86_compiled_binary.py", - "tests/integration/test_nop_insertion_pe_real.py" + "tests/integration/test_mutations_x86_compiled_binary.py::test_x86_string_obfuscation_preserves_native_exit_code" ] }, { @@ -4054,6 +4598,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4062,6 +4607,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4070,6 +4616,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4078,6 +4625,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -4089,6 +4637,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4097,6 +4646,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4105,6 +4655,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4113,6 +4664,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4121,6 +4673,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4129,6 +4682,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4137,6 +4691,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4145,6 +4700,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4153,6 +4709,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4161,6 +4718,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4169,6 +4727,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4177,6 +4736,7 @@ "format": "ELF", "architecture": "x86-64", "declared_support": true, + "support_status": "experimental", "status": "evidenced", "evidence": [ "README.md", @@ -4188,6 +4748,7 @@ "format": "ELF", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4196,6 +4757,7 @@ "format": "ELF", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4204,6 +4766,7 @@ "format": "ELF", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4212,6 +4775,7 @@ "format": "PE", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4220,6 +4784,7 @@ "format": "PE", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4228,6 +4793,7 @@ "format": "PE", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4236,6 +4802,7 @@ "format": "PE", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4244,6 +4811,7 @@ "format": "Mach-O", "architecture": "x86-64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4252,6 +4820,7 @@ "format": "Mach-O", "architecture": "x86", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4260,6 +4829,7 @@ "format": "Mach-O", "architecture": "AArch64", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] }, @@ -4268,6 +4838,7 @@ "format": "Mach-O", "architecture": "ARM", "declared_support": false, + "support_status": null, "status": "not-supported", "evidence": [] } diff --git a/pyproject.toml b/pyproject.toml index 69e32c25..e88e0fba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,12 +66,12 @@ dev = [ "click>=8.3.3", "GitPython>=3.1.60", "idna>=3.15", - "msgpack>=1.2.1", + "msgpack>=1.2.3", "pygments>=2.20.0", "requests>=2.33.0", "urllib3>=2.7.0", "protobuf>=6.33.5", - "unicorn>=2.0.0; platform_system != \"Darwin\" or platform_machine != \"arm64\"", + "unicorn>=2.0.0", ] docs = [ "mkdocs>=1.5.0", @@ -98,7 +98,7 @@ syntia = [ ] devirtualization = [ "miasm>=0.1.0", - "unicorn>=2.0.0; platform_system != \"Darwin\" or platform_machine != \"arm64\"", + "unicorn>=2.0.0", ] machine-learning = [ "scikit-learn>=1.3.0", @@ -121,7 +121,7 @@ all = [ "pysmt>=0.9.0", "claripy>=9.3.2", "miasm>=0.1.0", - "unicorn>=2.0.0; platform_system != \"Darwin\" or platform_machine != \"arm64\"", + "unicorn>=2.0.0", "scikit-learn>=1.3.0", "torch>=2.0.0", "transformers>=4.30.0", diff --git a/r2morph/adapters/angr_import.py b/r2morph/adapters/angr_import.py index b8cc8455..a385f4eb 100644 --- a/r2morph/adapters/angr_import.py +++ b/r2morph/adapters/angr_import.py @@ -78,7 +78,9 @@ def import_angr_modules( with warnings.catch_warnings(): warnings.filterwarnings("ignore", message=_PY314_CTYPES_PACK_WARNING, category=DeprecationWarning) angr_module = importlib.import_module("angr") - claripy_module = importlib.import_module("claripy") if claripy else None + claripy_module = getattr(angr_module, "claripy", None) if claripy else None + if claripy and claripy_module is None: + claripy_module = importlib.import_module("claripy") project_class = angr_module.Project if project else None cfg_fast_class = importlib.import_module("angr.analyses").CFGFast if cfg_fast else None technique_class = ( diff --git a/r2morph/adapters/process.py b/r2morph/adapters/process.py index d6fdae2b..0c255e57 100644 --- a/r2morph/adapters/process.py +++ b/r2morph/adapters/process.py @@ -9,6 +9,7 @@ import shutil import signal from collections.abc import Mapping, Sequence +from contextlib import suppress from dataclasses import dataclass from pathlib import Path @@ -104,15 +105,40 @@ async def _run_process( try: stdout, stderr = await asyncio.wait_for(process.communicate(context.input_bytes), timeout) except TimeoutError: - if os.name == "posix": - os.killpg(process.pid, signal.SIGKILL) - else: - process.kill() - await process.communicate() + await _terminate_process_tree(process) raise ProcessTimeoutError(f"Process exceeded {timeout} seconds") from None return ProcessResult(stdout=stdout, stderr=stderr, returncode=process.returncode or 0) +async def _terminate_process_tree(process: asyncio.subprocess.Process) -> None: + if os.name == "posix": + os.killpg(process.pid, signal.SIGKILL) + else: + try: + killer = await asyncio.create_subprocess_exec( + "taskkill", + "/PID", + str(process.pid), + "/T", + "/F", + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + await killer.wait() + if killer.returncode != 0: + _kill_if_running(process) + except OSError: + _kill_if_running(process) + await process.communicate() + + +def _kill_if_running(process: asyncio.subprocess.Process) -> None: + if process.returncode is None: + with suppress(ProcessLookupError): + # The child can exit between taskkill's result and the fallback kill. + process.kill() + + def run_process( command: Sequence[str | Path], *, diff --git a/r2morph/analysis/enhanced_analyzer.py b/r2morph/analysis/enhanced_analyzer.py index dfc9cce9..65be7113 100644 --- a/r2morph/analysis/enhanced_analyzer.py +++ b/r2morph/analysis/enhanced_analyzer.py @@ -32,6 +32,7 @@ ) from r2morph.detection.anti_analysis_bypass import AntiAnalysisBypass from r2morph.detection.obfuscation_detector import ObfuscationDetector +from r2morph.utils.logging import observed_operation logger = logging.getLogger(__name__) @@ -183,47 +184,48 @@ def analyze(self, options: AnalysisOptions | None = None) -> AnalysisResults: options = AnalysisOptions() try: - self._load_binary() - self.run_detection() - self.display_detection_results(verbose=options.verbose) + with observed_operation(logger, "analysis.enhanced", event_name="analysis.enhanced"): + self._load_binary() + self.run_detection() + self.display_detection_results(verbose=options.verbose) - if options.detect_only: - return self.results + if options.detect_only: + return self.results - result = self.results.detection_result + result = self.results.detection_result - if options.bypass: - self.run_anti_analysis_bypass() + if options.bypass: + self.run_anti_analysis_bypass() - if result.control_flow_flattened or options.devirt: - self.run_cfo_simplification() + if result.control_flow_flattened or options.devirt: + self.run_cfo_simplification() - if options.iterative: - self.run_iterative_simplification(max_iterations=options.max_iterations, timeout=options.timeout) + if options.iterative: + self.run_iterative_simplification(max_iterations=options.max_iterations, timeout=options.timeout) - if options.symbolic and result.vm_detected: - self.run_symbolic_analysis() + if options.symbolic and result.vm_detected: + self.run_symbolic_analysis() - if options.dynamic: - self.run_dynamic_analysis() + if options.dynamic: + self.run_dynamic_analysis() - if options.rewrite: - self.run_binary_rewriting() + if options.rewrite: + self.run_binary_rewriting() - self.display_analysis_results() + self.display_analysis_results() - report = self.generate_report() + report = self.generate_report() - if self.output_dir: - self.save_report(report) + if self.output_dir: + self.save_report(report) - self.display_recommendations() + self.display_recommendations() - self.console.print("\n[bold green]Phase 2 Analysis Complete![/bold green]") - if self.output_dir: - self.console.print(f"Results saved to: {self.output_dir}") + self.console.print("\n[bold green]Phase 2 Analysis Complete![/bold green]") + if self.output_dir: + self.console.print(f"Results saved to: {self.output_dir}") - return self.results + return self.results finally: self._cleanup() diff --git a/r2morph/analysis/enhanced_analyzer_phases.py b/r2morph/analysis/enhanced_analyzer_phases.py index fa624b1f..bec0bcf2 100644 --- a/r2morph/analysis/enhanced_analyzer_phases.py +++ b/r2morph/analysis/enhanced_analyzer_phases.py @@ -2,10 +2,37 @@ from __future__ import annotations +import logging from importlib import import_module from pathlib import Path from typing import Any, cast +logger = logging.getLogger(__name__) + + +def _log_phase_failure(operation: str) -> None: + logger.warning( + "Enhanced analysis phase failed", + extra={ + "event_name": f"{operation}.failed", + "operation": operation, + "outcome": "degraded", + }, + exc_info=True, + ) + + +def _log_optional_phase_unavailable(operation: str, error: ImportError) -> None: + logger.warning( + "Optional analysis phase unavailable", + extra={ + "event_name": f"{operation}.unavailable", + "operation": operation, + "outcome": "degraded", + "error_type": type(error).__name__, + }, + ) + def run_cfo_simplification(binary: Any, console: Any, results: Any) -> int: """Run Control Flow Obfuscation simplification.""" @@ -30,6 +57,7 @@ def run_cfo_simplification(binary: Any, console: Any, results: Any) -> int: return total_reduction except Exception as e: + _log_phase_failure("analysis.cfo_simplification") console.print(f"[yellow]CFO simplification error: {e}[/yellow]") return 0 @@ -69,6 +97,7 @@ def run_iterative_simplification( return None except Exception as e: + _log_phase_failure("analysis.iterative_simplification") console.print(f"[yellow]Iterative simplification error: {e}[/yellow]") return None @@ -96,7 +125,8 @@ def run_symbolic_analysis(binary: Any, console: Any, results: Any) -> int | None return handlers_count return None - except ImportError: + except ImportError as error: + _log_optional_phase_unavailable("analysis.symbolic", error) console.print("[yellow]Symbolic execution not available (missing angr)[/yellow]") return None @@ -112,7 +142,8 @@ def run_dynamic_analysis(console: Any) -> bool: console.print("Frida engine initialized") return True - except ImportError: + except ImportError as error: + _log_optional_phase_unavailable("analysis.dynamic", error) console.print("[yellow]Dynamic instrumentation not available (missing frida)[/yellow]") return False @@ -149,9 +180,18 @@ def run_binary_rewriting( results.rewrite_output = str(output_path) return str(output_path) + logger.warning( + "Binary rewriting did not produce output", + extra={ + "event_name": "analysis.binary_rewriting.failed", + "operation": "analysis.binary_rewriting", + "outcome": "degraded", + }, + ) console.print("Binary rewriting failed") return None except Exception as e: + _log_phase_failure("analysis.binary_rewriting") console.print(f"[yellow]Binary rewriting error: {e}[/yellow]") return None diff --git a/r2morph/analysis/symbolic/angr_bridge.py b/r2morph/analysis/symbolic/angr_bridge.py index bf7e89e6..60cc1119 100644 --- a/r2morph/analysis/symbolic/angr_bridge.py +++ b/r2morph/analysis/symbolic/angr_bridge.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: import angr + import claripy from angr import Project from angr.analyses import CFGFast @@ -21,9 +22,10 @@ else: from r2morph.adapters.angr_import import import_angr_modules - _angr_import = import_angr_modules(project=True, cfg_fast=True) + _angr_import = import_angr_modules(claripy=True, project=True, cfg_fast=True) ANGR_AVAILABLE = _angr_import.available angr = _angr_import.angr + claripy = _angr_import.claripy Project = _angr_import.project CFGFast = _angr_import.cfg_fast diff --git a/r2morph/analysis/symbolic/constraint_solver_analysis.py b/r2morph/analysis/symbolic/constraint_solver_analysis.py index 03b9ffe1..b4cecdf5 100644 --- a/r2morph/analysis/symbolic/constraint_solver_analysis.py +++ b/r2morph/analysis/symbolic/constraint_solver_analysis.py @@ -120,7 +120,16 @@ def simplify_mba_expression( confidence=max(0.1, confidence), ) except Exception as exc: - logger.error(f"Error simplifying MBA expression: {exc}") + logger.error( + "MBA constraint simplification failed", + extra={ + "event_name": "analysis.mba_constraint_simplification.failed", + "operation": "analysis.mba_constraint_simplification", + "outcome": "failure", + "error_type": type(exc).__name__, + }, + ) + logger.debug("MBA constraint simplification detail: %s", exc) return SolverResult(satisfiable=False, solving_time=time.time() - start_time, solver_used="z3") diff --git a/r2morph/analysis/symbolic/syntia_integration.py b/r2morph/analysis/symbolic/syntia_integration.py index 15d97cb2..cd69b537 100644 --- a/r2morph/analysis/symbolic/syntia_integration.py +++ b/r2morph/analysis/symbolic/syntia_integration.py @@ -209,7 +209,7 @@ def simplify_mba_with_syntia(self, mba_expression: str, variables: set[str]) -> Returns: Simplified expression or None if simplification failed """ - logger.info(f"Simplifying MBA expression: {mba_expression}") + logger.debug("Simplifying MBA expression: %s", mba_expression) # Try systematic simplification rules. Syntia-based program synthesis # would live alongside this branch but is not yet wired up; the diff --git a/r2morph/cli.py b/r2morph/cli.py index e68e65d3..9df8e3f9 100644 --- a/r2morph/cli.py +++ b/r2morph/cli.py @@ -50,7 +50,7 @@ from r2morph.core.support import PRODUCT_SUPPORT from r2morph.reporting.cli_commands import ReportCommandOptions, handle_report_command from r2morph.reporting.report_context import PassClassFilters -from r2morph.utils.logging import observed_operation, sanitize_log_text, setup_logging +from r2morph.utils.logging import logging_context, observed_operation, sanitize_log_text, setup_logging logger = logging.getLogger(__name__) @@ -113,7 +113,7 @@ def _handle_main(options: MainCommandOptions) -> None: r2morph functions input.exe r2morph morph input.exe -m nop """ - setup_logging("DEBUG" if (options.verbose or options.debug) else "INFO") + setup_logging("DEBUG" if (options.verbose or options.debug) else None) if options.ctx.invoked_subcommand is not None: return @@ -160,10 +160,13 @@ def analyze( """ Analyze a binary and display statistics. """ - setup_logging("DEBUG" if verbose else "INFO") + setup_logging("DEBUG" if verbose else None) - with console.status("[bold green]Analyzing binary..."): - try: + try: + with ( + observed_operation(logger, "cli.analyze", event_name="cli.analyze"), + console.status("[bold green]Analyzing binary..."), + ): binary_analyzer_cls = _load_binary_analyzer() with MorphEngine() as engine: engine.load_binary(binary).analyze() @@ -178,12 +181,11 @@ def analyze( table.add_row(label, value) console.print(table) - - except typer.Exit: - raise - except Exception as e: - console.print(f"[bold red]Error:[/bold red] {sanitize_log_text(str(e))}") - raise typer.Exit(1) from e + except typer.Exit: + raise + except Exception as e: + console.print(f"[bold red]Error:[/bold red] {sanitize_log_text(str(e))}") + raise typer.Exit(1) from e def _handle_enhanced_analysis(options: EnhancedAnalysisOptions) -> None: @@ -198,45 +200,60 @@ def _handle_enhanced_analysis(options: EnhancedAnalysisOptions) -> None: - Binary rewriting and reconstruction - Anti-analysis bypass framework """ - setup_logging("DEBUG" if options.verbose else "INFO") - - analyzer_module = import_module("r2morph.analysis.enhanced_analyzer") - models_module = import_module("r2morph.analysis.enhanced_analyzer_models") - enhanced_analysis_orchestrator = analyzer_module.EnhancedAnalysisOrchestrator - check_enhanced_dependencies = analyzer_module.check_enhanced_dependencies - analysis_options = models_module.AnalysisOptions - - if not check_enhanced_dependencies(): - console.print("[bold red]Error:[/bold red] Enhanced analysis requires additional dependencies.") - console.print("Install with: [cyan]pip install 'r2morph[enhanced]'[/cyan]") - raise typer.Exit(1) - - with console.status("[bold green]Analyzing obfuscated binary..."): - try: - analysis_configuration = analysis_options( - verbose=options.verbose, - detect_only=options.detect_only, - symbolic=options.symbolic, - dynamic=options.dynamic, - devirt=options.devirt, - iterative=options.iterative, - rewrite=options.rewrite, - bypass=options.bypass, + setup_logging("DEBUG" if options.verbose else None) + + with observed_operation( + logger, + "cli.enhanced_analysis.initialize", + event_name="cli.enhanced_analysis.initialize", + ) as correlation_id: + analyzer_module = import_module("r2morph.analysis.enhanced_analyzer") + models_module = import_module("r2morph.analysis.enhanced_analyzer_models") + enhanced_analysis_orchestrator = analyzer_module.EnhancedAnalysisOrchestrator + check_enhanced_dependencies = analyzer_module.check_enhanced_dependencies + analysis_options = models_module.AnalysisOptions + + with logging_context(correlation_id=correlation_id, operation="cli.enhanced_analysis"): + if not check_enhanced_dependencies(): + logger.warning( + "Enhanced analysis dependencies unavailable", + extra={ + "event_name": "cli.enhanced_analysis.unavailable", + "operation": "cli.enhanced_analysis", + "outcome": "degraded", + "status": "unavailable", + }, ) + console.print("[bold red]Error:[/bold red] Enhanced analysis requires additional dependencies.") + console.print("Install with: [cyan]pip install 'r2morph[enhanced]'[/cyan]") + raise typer.Exit(1) - orchestrator = enhanced_analysis_orchestrator( - binary_path=options.binary, - output_dir=options.output, - console=console, - ) + with console.status("[bold green]Analyzing obfuscated binary..."): + try: + analysis_configuration = analysis_options( + verbose=options.verbose, + detect_only=options.detect_only, + symbolic=options.symbolic, + dynamic=options.dynamic, + devirt=options.devirt, + iterative=options.iterative, + rewrite=options.rewrite, + bypass=options.bypass, + ) + + orchestrator = enhanced_analysis_orchestrator( + binary_path=options.binary, + output_dir=options.output, + console=console, + ) - orchestrator.analyze(analysis_configuration) + orchestrator.analyze(analysis_configuration) - except Exception as e: - console.print(f"[bold red]Error:[/bold red] {sanitize_log_text(str(e))}") - if options.verbose: - console.print(sanitize_log_text(traceback.format_exc())) - raise typer.Exit(1) from e + except Exception as e: + console.print(f"[bold red]Error:[/bold red] {sanitize_log_text(str(e))}") + if options.verbose: + console.print(sanitize_log_text(traceback.format_exc())) + raise typer.Exit(1) from e analyze_enhanced = CommandCallback("analyze_enhanced", EnhancedAnalysisOptions, _handle_enhanced_analysis) @@ -252,10 +269,13 @@ def functions( """ List functions in a binary. """ - setup_logging("DEBUG" if verbose else "INFO") + setup_logging("DEBUG" if verbose else None) - with console.status("[bold green]Loading binary..."): - try: + try: + with ( + observed_operation(logger, "cli.functions", event_name="cli.functions"), + console.status("[bold green]Loading binary..."), + ): binary_analyzer_cls = _load_binary_analyzer() with MorphEngine() as engine: engine.load_binary(binary).analyze() @@ -276,10 +296,9 @@ def functions( notice = build_function_limit_notice(limit, len(funcs)) if notice is not None: console.print(f"\n[yellow]{notice}[/yellow]") - - except Exception as e: - console.print(f"[bold red]Error:[/bold red] {sanitize_log_text(str(e))}") - raise typer.Exit(1) from e + except Exception as e: + console.print(f"[bold red]Error:[/bold red] {sanitize_log_text(str(e))}") + raise typer.Exit(1) from e def _handle_morph(options: MorphCommandOptions) -> None: @@ -291,7 +310,7 @@ def _handle_morph(options: MorphCommandOptions) -> None: r2morph mutate binary.exe -m nop -m substitute --report report.json r2morph mutate binary.exe --cache # Enable caching for faster repeated runs """ - setup_logging("DEBUG" if options.verbose else "INFO") + setup_logging("DEBUG" if options.verbose else None) output = options.output or options.binary.parent / f"{options.binary.stem}_morphed{options.binary.suffix}" @@ -363,7 +382,17 @@ def _handle_validate(options: ValidateCommandOptions) -> None: compare_files=options.compare_files, normalize_whitespace=options.normalize_whitespace, ) - result = validator.validate(options.original, options.mutated) + with observed_operation(logger, "cli.validate", event_name="cli.validate"): + result = validator.validate(options.original, options.mutated) + logger.info( + "Runtime validation result", + extra={ + "event_name": "cli.validate.result", + "operation": "cli.validate", + "outcome": "success" if result.passed else "failure", + "status": "passed" if result.passed else "failed", + }, + ) console.print_json(json.dumps(result.to_dict())) raise typer.Exit(0 if result.passed else 1) @@ -380,40 +409,42 @@ def diff( """ Show a lightweight diff summary between two binaries. """ - diff_analyzer_cls = _load_diff_analyzer() - analyzer = diff_analyzer_cls() - result = analyzer.compare(original, mutated) - console.print_json(json.dumps(result.__dict__)) + with observed_operation(logger, "cli.diff", event_name="cli.diff"): + diff_analyzer_cls = _load_diff_analyzer() + analyzer = diff_analyzer_cls() + result = analyzer.compare(original, mutated) + console.print_json(json.dumps(result.__dict__)) raise typer.Exit(0) def _handle_report(options: ReportViewOptions) -> None: """Display a previously generated engine report.""" - handle_report_command( - options.report_file, - ReportCommandOptions( - only_pass=options.only_pass, - only_status=options.only_status, - only_mismatches=options.only_mismatches, - summary_only=options.summary_only, - output=options.output, - require_results=options.require_results, - min_severity=options.min_severity, - only_expected_severity=options.only_expected_severity, - only_pass_failure=options.only_pass_failure, - only_degraded=options.only_degraded, - only_failed_gates=options.only_failed_gates, - pass_classes=PassClassFilters( - only_risky_passes=options.only_risky_passes, - only_structural_risk=options.only_structural_risk, - only_symbolic_risk=options.only_symbolic_risk, - only_uncovered_passes=options.only_uncovered_passes, - only_covered_passes=options.only_covered_passes, - only_clean_passes=options.only_clean_passes, + with observed_operation(logger, "cli.report", event_name="cli.report"): + handle_report_command( + options.report_file, + ReportCommandOptions( + only_pass=options.only_pass, + only_status=options.only_status, + only_mismatches=options.only_mismatches, + summary_only=options.summary_only, + output=options.output, + require_results=options.require_results, + min_severity=options.min_severity, + only_expected_severity=options.only_expected_severity, + only_pass_failure=options.only_pass_failure, + only_degraded=options.only_degraded, + only_failed_gates=options.only_failed_gates, + pass_classes=PassClassFilters( + only_risky_passes=options.only_risky_passes, + only_structural_risk=options.only_structural_risk, + only_symbolic_risk=options.only_symbolic_risk, + only_uncovered_passes=options.only_uncovered_passes, + only_covered_passes=options.only_covered_passes, + only_clean_passes=options.only_clean_passes, + ), + output_format=options.output_format, ), - output_format=options.output_format, - ), - ) + ) report = CommandCallback("report", ReportViewOptions, _handle_report) @@ -444,7 +475,11 @@ def cache( r2morph cache --clear --path /custom/cache # Clear specific cache directory """ try: - handle_cache_command(clear=clear, stats=stats, path=path, console=console) + if clear or stats: + with observed_operation(logger, "cli.cache", event_name="cli.cache"): + handle_cache_command(clear=clear, stats=stats, path=path, console=console) + else: + handle_cache_command(clear=clear, stats=stats, path=path, console=console) except SystemExit as exc: code = exc.code raise typer.Exit(code if isinstance(code, int) else (0 if code is None else 1)) from exc @@ -467,7 +502,7 @@ def main() -> None: args = parser.parse_args(argv) input_file = Path(args.input_opt or args.input_file) output_file = Path(args.output_opt or args.output_file) if (args.output_opt or args.output_file) else None - setup_logging("DEBUG" if (args.verbose or args.debug) else "INFO") + setup_logging("DEBUG" if (args.verbose or args.debug) else None) with observed_operation(logger, "cli.simple_mode", event_name="cli.simple_mode"): _run_simple_mode( SimpleModeRequest( diff --git a/r2morph/cli_workflows.py b/r2morph/cli_workflows.py index b0d9b07c..76376447 100644 --- a/r2morph/cli_workflows.py +++ b/r2morph/cli_workflows.py @@ -110,7 +110,7 @@ def _add_mutations( def _run_simple_mode(request: SimpleModeRequest) -> None: - setup_logging("DEBUG" if (request.verbose or request.debug) else "INFO") + setup_logging("DEBUG" if (request.verbose or request.debug) else None) output_file = request.output_file if output_file is None: diff --git a/r2morph/core/support.py b/r2morph/core/support.py index b55e5c59..712299af 100644 --- a/r2morph/core/support.py +++ b/r2morph/core/support.py @@ -74,7 +74,7 @@ def _normalize_architecture_name(architecture: str, bits: int | None = None) -> if lowered in {"arm64", "aarch64", "arm64e"}: return "arm64" if lowered in {"arm", "armv7", "armv7l", "armv7a", "thumb"}: - return "arm64" if lowered == "arm" and bits == ARCH_BITS_64 else "arm" + return "arm64" if lowered == "arm" and bits == ARCH_BITS_64 else "arm32" return normalized_arch diff --git a/r2morph/devirtualization/binary_rewriter.py b/r2morph/devirtualization/binary_rewriter.py index 61a3ef3d..7abe5848 100644 --- a/r2morph/devirtualization/binary_rewriter.py +++ b/r2morph/devirtualization/binary_rewriter.py @@ -498,11 +498,29 @@ def _assemble_instructions(self, instructions: list[str]) -> bytes: try: encoding, _ = self.ks.asm(asm_code) except (keystone.KsError, TypeError, ValueError) as e: - logger.error("Assembly failed for %r: %s; refusing to fabricate bytes", asm_code, e) + logger.error( + "Assembly failed; refusing to fabricate bytes", + extra={ + "event_name": "devirtualization.assembly.failed", + "operation": "devirtualization.assemble", + "outcome": "failure", + "error_type": type(e).__name__, + }, + ) + logger.debug("Assembly failed for %r: %s", asm_code, e) return b"" if not encoding: - logger.error("Assembler produced no encoding for %r; refusing to fabricate bytes", asm_code) + logger.error( + "Assembler produced no encoding; refusing to fabricate bytes", + extra={ + "event_name": "devirtualization.assembly.failed", + "operation": "devirtualization.assemble", + "outcome": "failure", + "error_code": "empty_encoding", + }, + ) + logger.debug("Assembler produced no encoding for %r", asm_code) return b"" return bytes(encoding) diff --git a/r2morph/devirtualization/mba_solver.py b/r2morph/devirtualization/mba_solver.py index b178ff1a..4cac61b2 100644 --- a/r2morph/devirtualization/mba_solver.py +++ b/r2morph/devirtualization/mba_solver.py @@ -224,7 +224,16 @@ def simplify_mba(self, expression: str, method: str = "auto") -> SimplificationR self.stats["z3_simplifications"] += 1 except Exception as e: - logger.error(f"Error simplifying MBA expression: {e}") + logger.error( + "MBA expression simplification failed", + extra={ + "event_name": "analysis.mba_simplification.failed", + "operation": "analysis.mba_simplification", + "outcome": "failure", + "error_type": type(e).__name__, + }, + ) + logger.debug("MBA expression simplification detail: %s", e) result.success = False result.solving_time = time.time() - start_time diff --git a/r2morph/mutations/block_reordering.py b/r2morph/mutations/block_reordering.py index 0de46952..6ff65b29 100644 --- a/r2morph/mutations/block_reordering.py +++ b/r2morph/mutations/block_reordering.py @@ -24,6 +24,8 @@ from r2morph.mutations.block_reordering_relocation import reorder_function_blocks logger = logging.getLogger(__name__) +_X86_64_BITS = 64 +_X86_BITS = 32 def _read_exception_frames(binary: Any) -> tuple[dict[int, Any] | None, str | None]: @@ -69,6 +71,17 @@ def __init__(self, config: dict[str, Any] | None = None): self.probability = self.config.get("probability", 0.3) self.max_functions = self.config.get("max_functions", 10) self.preserve_fallthrough = self.config.get("preserve_fallthrough", True) + self.set_support( + formats=("ELF", "Mach-O", "PE"), + architectures=("x86_64",), + validators=("structural", "runtime"), + stability="stable", + notes=( + "ELF x86-32 remains preview pending broader native corpus evidence.", + "Mach-O x86_64 remains preview pending broader native corpus evidence.", + "PE x86_64 remains preview pending broader native corpus evidence.", + ), + ) def _can_reorder_function(self, func: dict[str, Any], blocks: list[dict[str, Any]]) -> bool: return can_reorder_function(func, blocks) @@ -126,6 +139,30 @@ def apply(self, binary: Any) -> dict[str, Any]: self._reset_random() functions = binary.get_functions() + arch_info = binary.get_arch_info() + binary_format = str(arch_info.get("format", "")).upper() + architecture = str(arch_info.get("arch", "")).lower() + bits = arch_info.get("bits") + supported_format = ( + binary_format.startswith("ELF") + or binary_format.startswith("MACH") + or (binary_format.startswith("PE") and bits == _X86_64_BITS) + ) + if not (supported_format and architecture == "x86" and bits in {_X86_BITS, _X86_64_BITS}): + logger.warning( + "Block reordering skipped for unsupported target: format=%s arch=%s bits=%s", + binary_format, + architecture, + bits, + ) + return { + "mutations_applied": 0, + "functions_mutated": 0, + "total_blocks_reordered": 0, + "total_functions": len(functions), + "functions_processed": 0, + } + exception_frames, exception_read_error = _read_exception_frames(binary) functions_mutated = 0 total_blocks_reordered = 0 diff --git a/r2morph/mutations/block_reordering_relocation.py b/r2morph/mutations/block_reordering_relocation.py index 86a7a3ac..62504637 100644 --- a/r2morph/mutations/block_reordering_relocation.py +++ b/r2morph/mutations/block_reordering_relocation.py @@ -75,7 +75,7 @@ def reorder_function_blocks( func_end = ordered[-1]["addr"] + ordered[-1]["size"] budget = func_end - func_start plans = _build_block_plans(binary, ordered, func_start) - new_order = _draw_new_order(ordered, rng) + new_order = _draw_new_order(ordered, plans, rng) if new_order is None: raise _BailOutError("shuffle kept the original order") layout = _lay_out(binary, new_order, plans, func_start, budget) @@ -245,15 +245,25 @@ def _fallthrough_target( def _draw_new_order( ordered: list[dict[str, Any]], + plans: dict[int, _BlockPlan], rng: Any, ) -> list[dict[str, Any]] | None: - """Pin the entry block and shuffle the rest; None if order is unchanged.""" - tail = ordered[1:] + """Keep fallthrough chains intact while shuffling independent block groups.""" + groups: list[list[dict[str, Any]]] = [] + group: list[dict[str, Any]] = [] + for index, block in enumerate(ordered): + group.append(block) + next_addr = ordered[index + 1]["addr"] if index + 1 < len(ordered) else None + if next_addr is None or plans[block["addr"]].fallthrough != next_addr: + groups.append(group) + group = [] + + tail = groups[1:] shuffled = list(tail) rng.shuffle(shuffled) if shuffled == tail: return None - return [ordered[0], *shuffled] + return [block for blocks in [groups[0], *shuffled] for block in blocks] def _lay_out( diff --git a/r2morph/mutations/constant_unfolding.py b/r2morph/mutations/constant_unfolding.py index f1f92d6b..999d65dd 100644 --- a/r2morph/mutations/constant_unfolding.py +++ b/r2morph/mutations/constant_unfolding.py @@ -233,7 +233,7 @@ def _try_unfold_instruction( ) applied = self._apply_single_unfold(binary, mutation) if applied: - logger.info("Unfolded constant: '%s' -> '%s' at 0x%x", disasm, "; ".join(unfolded), addr) + logger.debug("Unfolded constant: '%s' -> '%s' at 0x%x", disasm, "; ".join(unfolded), addr) return applied, is_constant, new_size - orig_size if applied else 0 except (ValueError, OSError, BrokenPipeError, RuntimeError) as exc: logger.debug("Failed to unfold constant at 0x%x: %s", addr, exc) diff --git a/r2morph/mutations/instruction_substitution.py b/r2morph/mutations/instruction_substitution.py index 3448efce..dc99615a 100644 --- a/r2morph/mutations/instruction_substitution.py +++ b/r2morph/mutations/instruction_substitution.py @@ -243,7 +243,13 @@ def _apply_equivalent( if not self._validate_substitution(binary, record, checkpoint): return False suffix = " (+ NOPs)" if use_padding else "" - logger.info(f"Substituted '{instruction.get('disasm')}' with '{choice.replacement}'{suffix} at 0x{address:x}") + logger.debug( + "Substituted '%s' with '%s'%s at 0x%x", + instruction.get("disasm"), + choice.replacement, + suffix, + address, + ) return True def apply(self, binary: Any) -> dict[str, Any]: diff --git a/r2morph/mutations/pattern_substitution.py b/r2morph/mutations/pattern_substitution.py index e2dacd3d..75a3f716 100644 --- a/r2morph/mutations/pattern_substitution.py +++ b/r2morph/mutations/pattern_substitution.py @@ -192,7 +192,12 @@ def _apply_edit( if self._validate_mutation_or_rollback(binary, record, mutation_checkpoint): return False - logger.info(f"Pattern-substituted '{original.get('disasm')}' with '{replacement_disasm}' at 0x{addr:x}") + logger.debug( + "Pattern-substituted '%s' with '%s' at 0x%x", + original.get("disasm"), + replacement_disasm, + addr, + ) return True def _assemble_replacement(self, binary: Any, func_addr: int, replacement: list[Any]) -> bytes | None: diff --git a/r2morph/mutations/short_jump_patching.py b/r2morph/mutations/short_jump_patching.py index 75418882..0e1a7b5f 100644 --- a/r2morph/mutations/short_jump_patching.py +++ b/r2morph/mutations/short_jump_patching.py @@ -387,7 +387,7 @@ def apply(self, binary: Any) -> dict[str, Any]: f"Found {len(rip_relative_found)} RIP-relative instructions in {functions_with_rip} functions" ) for item in rip_relative_found[:_RIP_RELATIVE_PREVIEW_COUNT]: - logger.warning(f" 0x{item['address']:x}: {item['disasm']} in {item['function']}") + logger.debug(" 0x%x: %s in %s", item["address"], item["disasm"], item["function"]) if len(rip_relative_found) > _RIP_RELATIVE_PREVIEW_COUNT: logger.warning(f" ... and {len(rip_relative_found) - _RIP_RELATIVE_PREVIEW_COUNT} more") diff --git a/r2morph/reporting/sarif_formatter.py b/r2morph/reporting/sarif_formatter.py index bfbbabe3..f1cb9310 100644 --- a/r2morph/reporting/sarif_formatter.py +++ b/r2morph/reporting/sarif_formatter.py @@ -58,7 +58,6 @@ def format(self, report_data: ReportData) -> SARIFReport: artifacts=artifacts or None, invocations=invocations or None, taxonomies=[taxonomy], - original_uri_base_ids={"SRCROOT": str(Path.cwd())}, ) ] ) diff --git a/r2morph/reporting/sarif_formatter_builders.py b/r2morph/reporting/sarif_formatter_builders.py index 8efc3a3a..b5daac86 100644 --- a/r2morph/reporting/sarif_formatter_builders.py +++ b/r2morph/reporting/sarif_formatter_builders.py @@ -106,12 +106,16 @@ def build_driver(tool_version: str, information_uri: str, rules: list[SARIFRule] return SARIFTool(driver=driver) +def _artifact_uri(path: str) -> str: + return path.replace("\\", "/").rsplit("/", maxsplit=1)[-1] + + def build_artifacts(report_data: ReportData) -> list[SARIFArtifact]: artifacts = [] - loc = SARIFArtifactLocation(uri=report_data.binary_path) + loc = SARIFArtifactLocation(uri=_artifact_uri(report_data.binary_path)) artifacts.append(SARIFArtifact(location=loc, mime_type="application/octet-stream")) if report_data.output_path: - loc = SARIFArtifactLocation(uri=report_data.output_path) + loc = SARIFArtifactLocation(uri=_artifact_uri(report_data.output_path)) artifacts.append(SARIFArtifact(location=loc, mime_type="application/octet-stream")) return artifacts diff --git a/r2morph/utils/logging.py b/r2morph/utils/logging.py index df2fc781..882205ab 100644 --- a/r2morph/utils/logging.py +++ b/r2morph/utils/logging.py @@ -16,6 +16,7 @@ from contextvars import ContextVar, Token from dataclasses import dataclass, replace from datetime import UTC, datetime +from itertools import islice from logging.handlers import RotatingFileHandler from pathlib import Path from typing import TextIO @@ -33,11 +34,15 @@ "access_token", "authorization", "cookie", + "client_secret", + "id_token", + "oauth_token", "password", "passwd", "private_key", "refresh_token", "secret", + "secret_key", "session", "session_id", "token", @@ -64,7 +69,8 @@ ) _CONTROL_CHARACTERS = re.compile(r"[\x00-\x1f\x7f]") _SECRET_ASSIGNMENT = re.compile( - r"(?i)\b(password|passwd|token|access_token|refresh_token|api[-_]?key|secret|authorization|cookie|session)" + r"(?i)\b(password|passwd|token|access_token|refresh_token|oauth[-_]?token|id[-_]?token|api[-_]?key|" + r"client[-_]?secret|secret[-_]?key|private[-_]?key|secret|authorization|cookie|session)" r"(\s*[:=]\s*)([^\s,;]+)" ) _BEARER_TOKEN = re.compile(r"(?i)\bbearer\s+[a-z0-9._~+/=-]+") @@ -165,29 +171,32 @@ def sanitize_log_text(value: str) -> str: def _safe_value(value: object, *, key: str | None = None) -> object: - if key is not None and key.lower() in _SENSITIVE_KEYS: + sensitive_key = re.split(r"[:=]", key, maxsplit=1)[0].strip().lower().replace("-", "_") if key is not None else None + if sensitive_key in _SENSITIVE_KEYS: result: object = "" elif isinstance(value, Path): - result = f"" + result = sanitize_log_text(f"") elif isinstance(value, bytes): digest = hashlib.sha256(value).hexdigest()[:12] result = {"size": len(value), "sha256_prefix": digest} elif isinstance(value, str): result = sanitize_log_text(value) elif isinstance(value, Mapping): - items = list(value.items()) + item_count = len(value) + items = list(islice(value.items(), _MAX_COLLECTION_ITEMS)) safe_mapping = { - str(item_key): _safe_value(item_value, key=str(item_key)) - for item_key, item_value in items[:_MAX_COLLECTION_ITEMS] + sanitize_log_text(str(item_key)): _safe_value(item_value, key=str(item_key)) + for item_key, item_value in items } - if len(items) > _MAX_COLLECTION_ITEMS: - safe_mapping["_truncated_items"] = len(items) - _MAX_COLLECTION_ITEMS + if item_count > _MAX_COLLECTION_ITEMS: + safe_mapping["_truncated_items"] = item_count - _MAX_COLLECTION_ITEMS result = safe_mapping elif isinstance(value, (list, tuple, set, frozenset)): - items = list(value) - safe_items = [_safe_value(item) for item in items[:_MAX_COLLECTION_ITEMS]] - if len(items) > _MAX_COLLECTION_ITEMS: - safe_items.append(f"") + item_count = len(value) + items = list(islice(value, _MAX_COLLECTION_ITEMS)) + safe_items = [_safe_value(item) for item in items] + if item_count > _MAX_COLLECTION_ITEMS: + safe_items.append(f"") result = safe_items elif value is None or isinstance(value, (bool, int, float)): result = value diff --git a/r2morph/validation/binary_region_bridges.py b/r2morph/validation/binary_region_bridges.py index 3a093ece..1ecd307d 100644 --- a/r2morph/validation/binary_region_bridges.py +++ b/r2morph/validation/binary_region_bridges.py @@ -141,7 +141,12 @@ def setup_symbolic_bridges( "symbolic_binary_check_performed": False, "symbolic_binary_reason": "angr module not available", } - claripy = import_module("claripy") + claripy = getattr(bridge_module, "claripy", None) + if claripy is None: + return { + "symbolic_binary_check_performed": False, + "symbolic_binary_reason": "claripy module not available", + } options = angr_module.options return (original_bridge, mutated_bridge, angr_module, claripy, options) diff --git a/r2morph/validation/shellcode_observables.py b/r2morph/validation/shellcode_observables.py index fbe3993e..b70660a9 100644 --- a/r2morph/validation/shellcode_observables.py +++ b/r2morph/validation/shellcode_observables.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass -from importlib import import_module from typing import Any from r2morph.core.binary import Binary @@ -32,6 +31,12 @@ def compare_instruction_substitution_observables( "symbolic_observable_check_performed": False, "symbolic_observable_reason": "angr module not available", } + claripy_module = getattr(bridge_module, "claripy", None) + if claripy_module is None: + return { + "symbolic_observable_check_performed": False, + "symbolic_observable_reason": "claripy module not available", + } config = _observables_arch_config(binary.get_arch_info()) if config is None: @@ -42,7 +47,7 @@ def compare_instruction_substitution_observables( shellcode_arch, observables = config execution = _ObservableExecution( angr=angr_module, - claripy=import_module("claripy"), + claripy=claripy_module, options=angr_module.options, shellcode_arch=shellcode_arch, observables=observables, diff --git a/r2morph/validation/shellcode_transition.py b/r2morph/validation/shellcode_transition.py index d1d46ad4..955ba58e 100644 --- a/r2morph/validation/shellcode_transition.py +++ b/r2morph/validation/shellcode_transition.py @@ -3,7 +3,6 @@ from __future__ import annotations from dataclasses import dataclass -from importlib import import_module from typing import Any from r2morph.core.binary import Binary @@ -35,6 +34,12 @@ def compare_instruction_substitution_transition( "symbolic_transition_check_performed": False, "symbolic_transition_reason": "angr module not available", } + claripy_module = getattr(bridge_module, "claripy", None) + if claripy_module is None: + return { + "symbolic_transition_check_performed": False, + "symbolic_transition_reason": "claripy module not available", + } config = _transition_arch_config(binary.get_arch_info()) if config is None: @@ -45,8 +50,7 @@ def compare_instruction_substitution_transition( shellcode_arch, stack_reg, bit_width = config options = angr_module.options - claripy = import_module("claripy") - runtime = TransitionRuntime(angr_module, claripy, options, shellcode_arch, stack_reg, bit_width) + runtime = TransitionRuntime(angr_module, claripy_module, options, shellcode_arch, stack_reg, bit_width) compared_regions: list[dict[str, Any]] = [] mismatches: list[dict[str, Any]] = [] diff --git a/requirements.txt b/requirements.txt index 3bab487d..92d946b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,12 +25,12 @@ ipython>=8.12.0 click>=8.3.3 GitPython>=3.1.60 idna>=3.15 -msgpack>=1.2.1 +msgpack>=1.2.3 pygments>=2.20.0 requests>=2.33.0 urllib3>=2.7.0 protobuf>=6.33.5 -unicorn>=2.0.0; platform_system != "Darwin" or platform_machine != "arm64" +unicorn>=2.0.0 # Analysis/test extras exercised by the suite angr>=9.3.2; platform_system != "Windows" diff --git a/scripts/check_critical_skips.py b/scripts/check_critical_skips.py new file mode 100644 index 00000000..8a969bc0 --- /dev/null +++ b/scripts/check_critical_skips.py @@ -0,0 +1,28 @@ +"""Fail when a critical smoke-test JUnit report contains skipped cases.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from defusedxml import ElementTree + + +def skipped_test_count(report_path: Path) -> int: + return len(ElementTree.parse(report_path).getroot().findall(".//skipped")) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path, help="pytest JUnit XML report") + skipped = skipped_test_count(parser.parse_args().report) + if skipped: + print(f"critical smoke report contains {skipped} skipped test(s)", file=sys.stderr) + return 1 + print("critical smoke report contains no skipped tests") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_release_contract.py b/scripts/check_release_contract.py index e7fe9676..16a37f1d 100644 --- a/scripts/check_release_contract.py +++ b/scripts/check_release_contract.py @@ -6,6 +6,11 @@ import sys import tomllib +try: + from scripts.one_zero_release_gate import check_one_zero_release_gate as _check_one_zero_release_gate +except ModuleNotFoundError: + from one_zero_release_gate import check_one_zero_release_gate as _check_one_zero_release_gate + try: from scripts.release_contract_common import ( _BANNED_BINARY_NINJA_OMISSION_PHRASES, @@ -39,6 +44,7 @@ ) except ModuleNotFoundError: from release_contract_common import ( + ROOT, _check_version, ) @@ -139,6 +145,7 @@ _check_corpus_workflows, _check_release_recovery_workflow, _check_release_workflow, + _check_windows_download_contract, ) except ModuleNotFoundError: from release_contract_workflows import ( @@ -146,6 +153,7 @@ _check_corpus_workflows, _check_release_recovery_workflow, _check_release_workflow, + _check_windows_download_contract, ) __all__ = ( @@ -194,6 +202,7 @@ "_check_matrix", "_check_maturity_gap_evidence", "_check_maturity_profile_values", + "_check_one_zero_release_gate", "_check_parity_gap_evidence", "_check_pass_maturity_gap_summary", "_check_pass_selection_contract", @@ -215,6 +224,7 @@ "_check_vm_semantic_campaign_capabilities", "_check_vm_semantic_fixture_coverage", "_check_vm_semantic_gap_evidence", + "_check_windows_download_contract", "_forbidden_release_claims", "_load_matrix", "_validate_adversarial_benchmark_artifact", @@ -232,6 +242,11 @@ def main() -> int: try: package_version = _check_version() + _check_one_zero_release_gate( + package_version, + ROOT / "docs" / "one-zero-readiness.json", + ROOT, + ) matrix = _load_matrix() _check_matrix(matrix, package_version) _check_readme_support_summary(matrix) @@ -254,6 +269,7 @@ def main() -> int: _check_documentation_claims() _check_release_blockers() _check_ci_contract() + _check_windows_download_contract() _check_release_workflow() _check_release_recovery_workflow() _check_corpus_workflows() diff --git a/scripts/independent_review.py b/scripts/independent_review.py index 02b7e609..ba81bc5f 100644 --- a/scripts/independent_review.py +++ b/scripts/independent_review.py @@ -5,7 +5,9 @@ import argparse import json +import re import sys +from datetime import date from pathlib import Path from typing import Any @@ -65,7 +67,7 @@ "PE, Mach-O, ARM, and AArch64 remain preview or experimental", "Binary Ninja is deliberately on hold", "anti-tamper and progressive bytecode protection", - "external human review records signoff", + "Independent human review records signoff", ) _CURRENT_CORPUS_REPORT = "protection-adversarial-corpus-2026-09-06-a727f304.json" _CURRENT_GHIDRA_REPORT = "protection-ghidra-corpus-2026-09-04-88258a05.json" @@ -81,6 +83,34 @@ def _check(name: str, passed: bool, detail: str) -> dict[str, object]: return {"name": name, "status": "passed" if passed else "failed", "detail": detail} +def _human_review_signoff(root: Path) -> dict[str, Any] | None: + path = root / "docs" / "human-review-signoff.json" + if not path.exists(): + return None + record = json.loads(path.read_text(encoding="utf-8")) + if ( + not isinstance(record, dict) + or record.get("schema_version") != 1 + or record.get("result") != "APPROVE" + or record.get("independent") is not True + or not isinstance(record.get("reviewer"), str) + or not isinstance(record.get("review_date_utc"), str) + or not isinstance(record.get("reviewed_commit"), str) + or re.fullmatch(r"[0-9a-f]{40}", record["reviewed_commit"]) is None + or record.get("findings") != [] + ): + raise ValueError("human review signoff record is invalid") + try: + date.fromisoformat(record["review_date_utc"]) + except ValueError as error: + raise ValueError("human review signoff date must use ISO format") from error + if not isinstance(record.get("independence_confirmation"), str) or not isinstance( + record.get("specialist_expertise_confirmation"), str + ): + raise ValueError("human review signoff must record its independence and expertise confirmations") + return record + + def _review_virtualization(root: Path) -> dict[str, object]: inventory = build_coverage_inventory(root / "fixtures" / "dataset") passed = inventory["unclassified"] == [] and inventory["covered_capability_count"] == inventory["capability_count"] @@ -115,7 +145,7 @@ def _review_release_blocker_ledger(root: Path) -> dict[str, object]: return _check( "release_blocker_ledger_scope", passed, - "release blocker ledger tracks per-pass, differential, VM, parity, benchmark, resistance, and signoff gaps", + "release blocker ledger tracks per-pass, differential, VM, parity, benchmark, resistance, and review signoff", ) @@ -234,11 +264,21 @@ def _review_vm_fail_closed_diagnostics(root: Path) -> dict[str, object]: def _review_vm_resistance_adversarial_scope(root: Path) -> dict[str, object]: handler = json.loads((root / "docs" / "protection-handler-clustering.json").read_text(encoding="utf-8")) bytecode = json.loads((root / "docs" / "protection-bytecode-grammar.json").read_text(encoding="utf-8")) + signoff = _human_review_signoff(root) passed = ( handler.get("adversarial_validation") == _EXPECTED_VM_ADVERSARIAL_VALIDATION and bytecode.get("adversarial_validation") == _EXPECTED_VM_ADVERSARIAL_VALIDATION + and signoff is not None + ) + return _check( + "vm_resistance_adversarial_scope", + passed, + ( + "automated resistance evidence is paired with an independent human signoff" + if signoff is not None + else "independent human signoff is missing" + ), ) - return _check("vm_resistance_adversarial_scope", passed, "VM resistance remains pending human adversarial review") def _review_benchmark(root: Path) -> dict[str, object]: @@ -591,6 +631,7 @@ def _review_fixtures(root: Path) -> dict[str, object]: def review(root: Path, adversarial_report: Path | None = None) -> dict[str, Any]: fuzz = run_campaign(cases=64, seed=20260827, max_payload=512) + human_signoff = _human_review_signoff(root) checks = [ _review_virtualization(root), _review_matrix(root), @@ -623,14 +664,23 @@ def review(root: Path, adversarial_report: Path | None = None) -> dict[str, Any] return { "schema_version": 1, "review_type": "automated-independent-second-pass", - "human_signoff": "not-attested", + "human_signoff": "attested" if human_signoff is not None else "not-attested", + "human_review": human_signoff, "release_decision": { - "status": "block-vm-milestone", - "reason": "automated evidence is not a substitute for external human review", + "status": "signoff-recorded" if human_signoff is not None else "block-vm-milestone", + "reason": ( + f"independent human review approved commit {human_signoff['reviewed_commit']}" + if human_signoff is not None + else "automated evidence is not a substitute for external human review" + ), }, "checks": checks, "passed": all(check["status"] == "passed" for check in checks), - "limitations": ["This artifact is not a substitute for an external human review."], + "limitations": ( + ["Human approval applies only to the reviewed commit."] + if human_signoff is not None + else ["This artifact is not a substitute for an external human review."] + ), } diff --git a/scripts/one_zero_release_gate.py b/scripts/one_zero_release_gate.py new file mode 100644 index 00000000..fa59d6f7 --- /dev/null +++ b/scripts/one_zero_release_gate.py @@ -0,0 +1,62 @@ +"""Gate stable-major releases on explicit, linked 1.0 evidence.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +_REQUIRED_CRITERIA = { + "cross_platform_parity", + "all_named_adversarial_analyzers", + "documented_resistance_scope", + "stable_api_cli", + "complete_compatibility_contract", + "published_correctness_results", + "published_cost_results", + "published_resistance_results", +} +_VALID_CRITERION_STATUSES = {"open", "in_progress", "complete"} + + +def check_one_zero_release_gate(version: str, readiness_path: Path, evidence_root: Path) -> None: + """Reject a stable-major package version until its evidence ledger is complete.""" + report = json.loads(readiness_path.read_text(encoding="utf-8")) + criteria = report.get("criteria") + if report.get("schema_version") != 1 or not isinstance(criteria, dict): + raise ValueError("1.0 readiness report has an invalid schema") + if set(criteria) != _REQUIRED_CRITERIA: + raise ValueError("1.0 readiness report must include every required criterion") + + blockers = _validate_criteria(criteria, evidence_root) + expected_status = "blocked" if blockers else "ready" + if report.get("status") != expected_status or report.get("blocking_criteria") != blockers: + raise ValueError("1.0 readiness status must match its criterion evidence") + + major = int(version.split(".", 1)[0]) + if major >= 1 and blockers: + raise ValueError(f"stable-major release blocked by incomplete 1.0 criteria: {', '.join(blockers)}") + + +def _validate_criteria(criteria: dict[str, Any], evidence_root: Path) -> list[str]: + blockers: list[str] = [] + for name in sorted(_REQUIRED_CRITERIA): + criterion = criteria[name] + if not isinstance(criterion, dict) or criterion.get("status") not in _VALID_CRITERION_STATUSES: + raise ValueError(f"1.0 readiness criterion has an invalid status: {name}") + evidence = criterion.get("evidence") + if not isinstance(evidence, list) or not evidence: + raise ValueError(f"1.0 readiness criterion needs linked evidence: {name}") + _validate_evidence_paths(name, evidence, evidence_root) + if criterion["status"] != "complete": + blockers.append(name) + return blockers + + +def _validate_evidence_paths(name: str, evidence: list[Any], evidence_root: Path) -> None: + for reference in evidence: + if not isinstance(reference, str) or not reference.strip(): + raise ValueError(f"1.0 readiness criterion has an invalid evidence path: {name}") + path = Path(reference) + if path.is_absolute() or ".." in path.parts or not (evidence_root / path).is_file(): + raise ValueError(f"1.0 readiness evidence file is missing or unsafe: {name}: {reference}") diff --git a/scripts/platform_evidence.py b/scripts/platform_evidence.py index 052cc884..35aaf46b 100644 --- a/scripts/platform_evidence.py +++ b/scripts/platform_evidence.py @@ -77,7 +77,6 @@ "Composition": "test_nop_then_instruction_substitution_arm64_preserves_native_output", "RegisterSubstitution": "test_register_substitution_arm64_preserves_generated_native_execution", "ConstantUnfolding": "test_constant_unfolding_arm64_preserves_native_output", - "StringObfuscation": "test_x86_string_obfuscation_preserves_native_exit_code", }, "windows-pe": { "NopInsertion": "test_nop_insertion_pe_x86_64_preserves_repaired_integrity", diff --git a/scripts/protection_maturity_baseline.py b/scripts/protection_maturity_baseline.py index 82de5cb0..46b82021 100644 --- a/scripts/protection_maturity_baseline.py +++ b/scripts/protection_maturity_baseline.py @@ -15,7 +15,7 @@ import tempfile import time from collections.abc import Mapping -from concurrent.futures import ProcessPoolExecutor +from concurrent.futures import ThreadPoolExecutor from functools import partial from pathlib import Path from types import FrameType @@ -50,6 +50,7 @@ from r2morph.mutations.short_jump_patching import ShortJumpPatchingPass from r2morph.mutations.stack_strings import StackStringsPass from r2morph.mutations.string_obfuscation import StringObfuscationPass +from scripts.protection_maturity_worker import FixtureWorkerTimeoutError, run_fixture_worker from tests.integration.elf_emulator import emulate_exit_code _ELF_MAGIC = b"\x7fELF" @@ -2475,11 +2476,10 @@ def record_fixture_results(results: tuple[tuple[str, dict[str, object]], ...]) - measurements[pass_name].append(measurement) if args.workers == 1: - results = (_measure_fixture_worker(task) for task in tasks) - for fixture_results in results: + for fixture_results in map(_measure_fixture_worker, tasks): record_fixture_results(fixture_results) return measurements - with ProcessPoolExecutor(max_workers=args.workers) as executor: + with ThreadPoolExecutor(max_workers=args.workers) as executor: for fixture_results in executor.map(_measure_fixture_worker, tasks): record_fixture_results(fixture_results) return measurements @@ -2488,7 +2488,10 @@ def record_fixture_results(results: tuple[tuple[str, dict[str, object]], ...]) - def _measure_fixture_worker( task: tuple[Path, range, Path, tuple[str, ...], tuple[tuple[str, ...], ...], float], ) -> tuple[tuple[str, dict[str, object]], ...]: - return _measure_fixture_task(task) + try: + return run_fixture_worker(task) + except FixtureWorkerTimeoutError as error: + raise _FixtureTimeout(str(error)) from None def _measure_fixture_task( @@ -2526,7 +2529,28 @@ def _emit_report(rendered: str, output: Path | None) -> None: print(rendered, end="") +def _run_fixture_worker_main() -> None: + payload = cast(dict[str, object], json.load(sys.stdin)) + task = ( + Path(str(payload["fixture"])), + range(int(payload["seed_start"]), int(payload["seed_stop"])), + Path(str(payload["output_root"])), + tuple(cast(list[str], payload["pass_names"])), + tuple(tuple(arguments) for arguments in cast(list[list[str]], payload["runtime_inputs"])), + float(payload["timeout_seconds"]), + ) + try: + results = _measure_fixture_task(task) + except _FixtureTimeout as error: + print(json.dumps({"status": "timeout", "error": str(error)})) + return + print(json.dumps({"status": "completed", "results": results})) + + def main() -> None: + if sys.argv[1:] == ["--_worker-task"]: + _run_fixture_worker_main() + return parser = argparse.ArgumentParser() parser.add_argument("fixtures", nargs="*", type=Path) parser.add_argument("--all", action="store_true", dest="all_fixtures") diff --git a/scripts/protection_maturity_worker.py b/scripts/protection_maturity_worker.py new file mode 100644 index 00000000..8e01bfb1 --- /dev/null +++ b/scripts/protection_maturity_worker.py @@ -0,0 +1,60 @@ +"""Isolate and supervise one protection maturity fixture worker.""" + +from __future__ import annotations + +import json +import sys +from collections.abc import Sequence +from pathlib import Path +from typing import cast + +from r2morph.adapters.process import ProcessContext, ProcessResult, ProcessTimeoutError, run_process + +_WORKER_GRACE_SECONDS = 5.0 + + +def run_supervised_command(command: Sequence[str], timeout: float, input_bytes: bytes, cwd: Path) -> ProcessResult: + """Run a command with a deadline that also terminates its process tree.""" + return run_process( + command, + timeout=timeout, + context=ProcessContext(input_bytes=input_bytes, cwd=cwd), + ) + + +def run_fixture_worker( + task: tuple[Path, range, Path, tuple[str, ...], tuple[tuple[str, ...], ...], float], +) -> tuple[tuple[str, dict[str, object]], ...]: + """Run a fixture in a process group so a hard timeout also kills its children.""" + fixture, seeds, output_root, pass_names, runtime_inputs, timeout_seconds = task + payload = { + "fixture": str(fixture), + "seed_start": seeds.start, + "seed_stop": seeds.stop, + "output_root": str(output_root), + "pass_names": pass_names, + "runtime_inputs": runtime_inputs, + "timeout_seconds": timeout_seconds, + } + worker_script = Path(__file__).with_name("protection_maturity_baseline.py") + command = [sys.executable, str(worker_script), "--_worker-task"] + try: + result = run_supervised_command( + command, + timeout_seconds + _WORKER_GRACE_SECONDS, + json.dumps(payload).encode(), + Path(__file__).resolve().parents[1], + ) + except ProcessTimeoutError: + raise FixtureWorkerTimeoutError(f"fixture {fixture.name!r} exceeded the hard worker timeout") from None + if result.returncode != 0: + raise RuntimeError(f"fixture worker for {fixture.name!r} exited with status {result.returncode}") + response = cast(dict[str, object], json.loads(result.stdout)) + if response.get("status") == "timeout": + raise FixtureWorkerTimeoutError(str(response["error"])) + raw_results = cast(list[list[object]], response["results"]) + return tuple((str(item[0]), cast(dict[str, object], item[1])) for item in raw_results) + + +class FixtureWorkerTimeoutError(TimeoutError): + """A fixture exceeded either its cooperative or hard worker deadline.""" diff --git a/scripts/release_contract_artifacts.py b/scripts/release_contract_artifacts.py index 4ada74f0..612676ec 100644 --- a/scripts/release_contract_artifacts.py +++ b/scripts/release_contract_artifacts.py @@ -153,13 +153,25 @@ def _check_vm_resistance_artifacts() -> None: def _validate_independent_review_artifact(report: dict[str, object]) -> None: if report.get("passed") is not True: raise ValueError("independent review artifact must pass") - if report.get("human_signoff") != "not-attested": - raise ValueError("independent review artifact must not claim human signoff") + human_review = report.get("human_review") + signoff = json.loads((ROOT / "docs" / "human-review-signoff.json").read_text(encoding="utf-8")) + if ( + report.get("human_signoff") != "attested" + or not isinstance(human_review, dict) + or human_review != signoff + or human_review.get("result") != "APPROVE" + or human_review.get("independent") is not True + or human_review.get("findings") != [] + ): + raise ValueError("independent review artifact must include approved independent human signoff") release_decision = report.get("release_decision") - if not isinstance(release_decision, dict) or release_decision.get("status") != "block-vm-milestone": - raise ValueError("independent review artifact must block the VM milestone without human signoff") - if release_decision.get("reason") != "automated evidence is not a substitute for external human review": - raise ValueError("independent review artifact must require external human review") + if ( + not isinstance(release_decision, dict) + or release_decision.get("status") != "signoff-recorded" + or release_decision.get("reason") + != f"independent human review approved commit {human_review['reviewed_commit']}" + ): + raise ValueError("independent review artifact must record the human review decision") checks = report.get("checks") if not isinstance(checks, list): raise ValueError("independent review artifact must contain checks") @@ -184,7 +196,10 @@ def _check_independent_review_artifact() -> None: def _check_independent_review_packet_claims() -> None: packet = " ".join((ROOT / "docs" / "independent-review-packet.md").read_text(encoding="utf-8").split()) for fragment in ( - "release_decision: block-vm-milestone", + "release_decision: signoff-recorded", + "Reviewer: Marc Rivero Lopez", + "Result: APPROVE", + "specialist experience in binary rewriting and virtualization", "memory, direct/indirect calls, returns, flags, FP/SIMD, varargs/ABI, unwinding, TLS/signals, " "SSA, and liveness", "unsupported instructions fail closed", diff --git a/scripts/release_contract_common.py b/scripts/release_contract_common.py index 33ce58e6..f47df75e 100644 --- a/scripts/release_contract_common.py +++ b/scripts/release_contract_common.py @@ -156,7 +156,7 @@ "adversarial_validation", "vm_resistance_blocker_totals", "anti-tamper and progressive bytecode protection", - "external human review records signoff", + "Independent human review records signoff", "human_signoff", "release_decision", ) diff --git a/scripts/release_contract_docs.py b/scripts/release_contract_docs.py index 494a6997..562f8589 100644 --- a/scripts/release_contract_docs.py +++ b/scripts/release_contract_docs.py @@ -236,18 +236,20 @@ def _check_changelog(package_version: str) -> None: changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") if f"## {package_version}" not in changelog: raise ValueError("changelog must contain the package version heading") - if package_version == _check_version() and "VM milestone blocked until external human review" not in changelog: - raise ValueError("changelog must preserve the VM milestone signoff blocker") + if package_version == _check_version() and "Independent human review approval is recorded" not in changelog: + raise ValueError("changelog must record the independent human review") def _check_readme_adversarial_summary() -> None: readme = (ROOT / "README.md").read_text(encoding="utf-8") report_name = "protection-adversarial-angr-local-2026-09-18-59898697.json" report = json.loads((ROOT / "docs" / report_name).read_text(encoding="utf-8")) + independent_review = json.loads((ROOT / "docs" / "independent-review.json").read_text(encoding="utf-8")) tools = report["tools"] completed = sum(1 for tool in tools if tool["status"] == "completed") unavailable = sum(1 for tool in tools if tool["status"] == "unavailable") binary_ninja = next(tool for tool in tools if tool["tool"] == "binary-ninja") + evidence_details = {check["name"]: check["detail"] for check in independent_review["checks"]} for fragment in ( f"{completed} completed analyzer slots", f"{unavailable} unavailable analyzer slots", @@ -255,6 +257,11 @@ def _check_readme_adversarial_summary() -> None: "Binary Ninja is an explicit analyzer slot", f"`binary-ninja` as {binary_ninja['status']}", binary_ninja["reason"], + evidence_details["triton_corpus_evidence"], + evidence_details["ida_corpus_evidence"], + evidence_details["ida_current_summary_evidence"], + "availability in one campaign does not fill unavailable rows in another", + "the all-named-analyzers 1.0 criterion remains open", ): if fragment not in readme: raise ValueError(f"README adversarial summary is missing: {fragment}") @@ -268,14 +275,14 @@ def _check_readme_vm_review_scope() -> None: for fragment in ( f"human_signoff: {review['human_signoff']}", f"release_decision: {release_decision['status']}", - "memory, direct/indirect calls, returns, flags, FP/SIMD, varargs/ABI, unwinding, TLS/signals, " - "SSA, and liveness paths remain explicit review scope", + "reviewed memory, direct/indirect calls, returns, flags, FP/SIMD, varargs/ABI, unwinding, TLS/signals, " + "SSA, and liveness paths", "unsupported instructions must fail closed", "partial protected functions", "instruction address, mnemonic, type, size, bounded opcode preview, capability, and reason", "VM ISA/opcode diversification, dispatcher/handler alternatives, superinstructions, anti-tamper, " "and progressive bytecode protection", - "human-review scope before the virtualizer milestone can be marked complete", + "Human independent review approved the reviewed commit", ): if fragment not in readme: raise ValueError(f"README VM review scope is missing: {fragment}") @@ -291,6 +298,21 @@ def _check_readme_vm_resistance_summary() -> None: readme = " ".join((ROOT / "README.md").read_text(encoding="utf-8").split()) handler = json.loads((ROOT / "docs" / "protection-handler-clustering.json").read_text(encoding="utf-8")) bytecode = json.loads((ROOT / "docs" / "protection-bytecode-grammar.json").read_text(encoding="utf-8")) + independent_analysis = json.loads( + (ROOT / "docs" / "protection-independent-analysis-2026-11-10.json").read_text(encoding="utf-8") + ) + readiness = json.loads((ROOT / "docs" / "one-zero-readiness.json").read_text(encoding="utf-8")) + required_limitations = ( + "does not establish general resistance to professional reverse engineering", + "does not replace the required three-analyzer release campaign", + ) + if not all(limitation in independent_analysis["limitations"] for limitation in required_limitations): + raise ValueError("independent analysis must retain its resistance scope limitations") + if ( + readiness["criteria"]["documented_resistance_scope"]["status"] != "complete" + or "documented_resistance_scope" in readiness["blocking_criteria"] + ): + raise ValueError("documented resistance scope must match its published limitation evidence") for fragment in ( f"{handler['seed_count']} seeds with {VM_HANDLER_COUNT} handlers per seed", f"{handler['cross_seed_exact_normalised_matches']} exact normalized cross-seed handler matches", @@ -298,6 +320,9 @@ def _check_readme_vm_resistance_summary() -> None: "target handler stride diversity", "protection-handler-clustering.json", "protection-bytecode-grammar.json", + "they do not measure generalized attack cost or establish resistance to professional reverse engineering", + "Published resistance results remain an open 1.0 criterion.", + *required_limitations, ): if fragment not in readme: raise ValueError(f"README VM resistance summary is missing: {fragment}") diff --git a/scripts/release_contract_matrix.py b/scripts/release_contract_matrix.py index a4858032..fe91e199 100644 --- a/scripts/release_contract_matrix.py +++ b/scripts/release_contract_matrix.py @@ -168,6 +168,22 @@ def _check_vm_resistance_gap_evidence(matrix: dict[str, object]) -> None: and not (ROOT / evidence_path).exists() ): raise ValueError(f"vm resistance gap evidence path is missing: {gap}.{evidence_path}") + signoff = json.loads((ROOT / "docs" / "human-review-signoff.json").read_text(encoding="utf-8")) + resolved = summary["vm_resistance_resolved_evidence"] + if not isinstance(resolved, dict) or "human-adversarial-validation" not in resolved: + raise ValueError("vm resistance resolved review evidence is missing") + human_review = resolved["human-adversarial-validation"] + if ( + human_review["status"] != "approved" + or human_review["evidence_quality"] != "independent-human-review" + or human_review["reviewed_commit"] != signoff["reviewed_commit"] + or human_review["reviewer"] != signoff["reviewer"] + or human_review["review_date_utc"] != signoff["review_date_utc"] + or human_review["specialist_expertise_confirmation"] != signoff["specialist_expertise_confirmation"] + or "docs/human-review-signoff.json" not in human_review["evidence"] + or gap_scope + ): + raise ValueError("vm resistance independent review evidence is stale") def _check_differential_gap_evidence(matrix: dict[str, object]) -> None: diff --git a/scripts/release_contract_workflows.py b/scripts/release_contract_workflows.py index 886a7068..8224e70f 100644 --- a/scripts/release_contract_workflows.py +++ b/scripts/release_contract_workflows.py @@ -36,15 +36,17 @@ def _check_ci_contract() -> None: "Install radare2 (Windows)", '"https://github.com/radareorg/radare2/releases/download/$version/$archiveName"', '"913e7d95e7458226a5e783240877f2c1d405398ab38ba176ce9b12e046e31f34"', - "Get-FileHash -Algorithm SHA256", + '& ".github/scripts/download_verified_artifact.ps1" -Url $url -Output $archive -Sha256 $expectedSha256', "Expand-Archive -Path $archive -DestinationPath $destination -Force", "$env:GITHUB_PATH", + "git build-essential clang gcc-arm-linux-gnueabihf qemu-user", "python -m pip install --force-reinstall dist/*.whl", 'tempfile.mkdtemp(prefix="r2morph-cross-platform-wheel-")', 'for name in ("tests", "fixtures", "scripts", "docs", "README.md", "pyproject.toml")', 'subprocess.run([sys.executable, "-c", "import r2morph; print(r2morph.__file__)"]', '"tests/unit/test_circular_imports.py"', '"tests/unit/test_cli_basic_commands.py::test_cli_version_function"', + '"tests/unit/test_public_surface_contract.py"', "python scripts/support_matrix.py --check docs/support-matrix.json", ): if fragment not in workflow: @@ -60,6 +62,7 @@ def _check_ci_contract() -> None: "test_targets = [", "x64_assembler_targets = [", 'if platform.machine().lower() in {"amd64", "x86_64"}:', + '"tests/unit/test_public_surface_contract.py"', '"tests/unit/test_pe_handler_parsing_contract.py"', '"tests/unit/test_pe_handler_struct_parse.py"', '"tests/unit/test_macho_parse_basic.py"', @@ -103,6 +106,29 @@ def _check_ci_contract() -> None: raise ValueError("coverage gate must be at least 75 percent") +def _check_windows_download_contract() -> None: + root = ROOT / ".github" + script = (root / "scripts" / "download_verified_artifact.ps1").read_text(encoding="utf-8") + for workflow_name in ("ci.yml", "differential-corpus.yml"): + workflow = (root / "workflows" / workflow_name).read_text(encoding="utf-8") + if ( + '& ".github/scripts/download_verified_artifact.ps1" -Url $url -Output $archive -Sha256 $expectedSha256' + not in workflow + ): + raise ValueError(f"{workflow_name} must use the verified Windows artifact downloader") + for fragment in ( + "$maximumAttempts = 3", + "Invoke-WebRequest -Uri $Url -OutFile $partialOutput -TimeoutSec 120", + "$statusCode -in @(408, 429)", + "$statusCode -ge 500", + "Start-Sleep -Seconds $attempt", + "Get-FileHash -Algorithm SHA256 -Path $partialOutput", + "Move-Item -LiteralPath $partialOutput -Destination $Output -Force", + ): + if fragment not in script: + raise ValueError(f"verified Windows artifact downloader is missing: {fragment}") + + def _check_release_workflow() -> None: workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8") for fragment in ( diff --git a/scripts/support_matrix.py b/scripts/support_matrix.py index 105ed229..0851bcec 100644 --- a/scripts/support_matrix.py +++ b/scripts/support_matrix.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Expand the declared pass support into an exhaustive matrix.""" +"""Expand explicit pass target cells into an exhaustive compatibility matrix.""" from __future__ import annotations @@ -109,7 +109,7 @@ "fp-simd", "ssa-liveness", ) -_VM_RESISTANCE_GAP_SCOPE = ("human-adversarial-validation",) +_VM_RESISTANCE_GAP_SCOPE: tuple[str, ...] = () _VM_RESISTANCE_GAP_EVIDENCE = { "human-adversarial-validation": { "status": "pending-human-adversarial-review", @@ -205,6 +205,7 @@ "string-obfuscation-instruction-catalogued", } FULL_EVIDENCE_PERCENT = 100.0 +_TARGET_SUPPORT_STATUSES = {"supported", "preview", "experimental"} def _add_count(counts: dict[str, int], value: str) -> None: @@ -408,6 +409,26 @@ def _vm_resistance_evidence_blockers() -> dict[str, object]: return {"vm_resistance_gap_scope": list(_VM_RESISTANCE_GAP_SCOPE)} +def _vm_resistance_resolved_evidence() -> dict[str, object]: + root = Path(__file__).resolve().parents[1] + signoff = json.loads((root / "docs" / "human-review-signoff.json").read_text(encoding="utf-8")) + return { + "human-adversarial-validation": { + "status": "approved", + "evidence_quality": "independent-human-review", + "reviewed_commit": signoff["reviewed_commit"], + "reviewer": signoff["reviewer"], + "review_date_utc": signoff["review_date_utc"], + "specialist_expertise_confirmation": signoff["specialist_expertise_confirmation"], + "evidence": [ + "docs/human-review-signoff.json", + "docs/independent-review-packet.md", + "docs/protection-vm-resistance-2026-09-22-e3a491b3.json", + ], + } + } + + def _vm_resistance_gap_evidence() -> dict[str, object]: return {gap: _VM_RESISTANCE_GAP_EVIDENCE[gap] for gap in _VM_RESISTANCE_GAP_SCOPE} @@ -653,6 +674,89 @@ def _parity_blocker_totals( } +def _cell_evidence(mutation_pass: dict[str, Any]) -> dict[tuple[str, str], list[str]]: + evidence_by_target: dict[tuple[str, str], list[str]] = {} + for cell in mutation_pass.get("evidence_cells", []): + target = (cell.get("format"), cell.get("architecture")) + references = cell.get("evidence", []) + if not all(isinstance(part, str) and part for part in target): + raise ValueError(f"invalid evidence target for pass {mutation_pass.get('name')}") + if not isinstance(references, list) or not all( + isinstance(reference, str) and reference for reference in references + ): + raise ValueError(f"invalid cell evidence references for pass {mutation_pass.get('name')}") + if target in evidence_by_target: + raise ValueError(f"duplicate evidence target for pass {mutation_pass.get('name')}: {target}") + evidence_by_target[target] = references + return evidence_by_target + + +def _validated_target_support_cell( + cell: object, + mutation_pass: dict[str, Any], + formats: tuple[str, ...], + architectures: tuple[str, ...], + official_target: tuple[object, object], +) -> tuple[tuple[str, str], str]: + """Validate one exact support cell and return its normalized key and status.""" + if not isinstance(cell, dict): + raise ValueError(f"invalid target cell for pass {mutation_pass.get('name')}") + binary_format = cell.get("format") + architecture = cell.get("architecture") + support_status = cell.get("support_status") + if not isinstance(binary_format, str) or not binary_format or not isinstance(architecture, str) or not architecture: + raise ValueError(f"invalid target cell for pass {mutation_pass.get('name')}") + target = (binary_format, architecture) + if binary_format not in formats or architecture not in architectures: + raise ValueError(f"target cell is outside declared dimensions for pass {mutation_pass.get('name')}: {target}") + if support_status not in _TARGET_SUPPORT_STATUSES: + raise ValueError(f"invalid target support status for pass {mutation_pass.get('name')}: {target}") + is_official = (binary_format, architecture) == official_target + if support_status == "supported" and not is_official: + raise ValueError(f"only the official target may be marked supported: {mutation_pass.get('name')}: {target}") + if is_official: + expected_status = "supported" if mutation_pass.get("stability") == "tier-1" else "experimental" + if support_status != expected_status: + raise ValueError(f"official target status must match pass stability: {mutation_pass.get('name')}") + return target, support_status + + +def _target_support_statuses( + mutation_pass: dict[str, Any], + formats: tuple[str, ...], + architectures: tuple[str, ...], + official_format: object, + official_architecture: object, +) -> dict[tuple[str, str], str]: + """Load explicit format-architecture support without forming a product.""" + statuses: dict[tuple[str, str], str] = {} + target_cells = mutation_pass.get("target_cells") + if not isinstance(target_cells, list) or not target_cells: + raise ValueError(f"pass needs explicit target_cells: {mutation_pass.get('name')}") + for cell in target_cells: + target, support_status = _validated_target_support_cell( + cell, + mutation_pass, + formats, + architectures, + (official_format, official_architecture), + ) + if target in statuses: + raise ValueError(f"duplicate target cell for pass {mutation_pass.get('name')}: {target}") + statuses[target] = support_status + + expected_formats = {binary_format for binary_format, _architecture in statuses} + expected_architectures = {architecture for _format, architecture in statuses} + if expected_formats != set(mutation_pass.get("formats", [])): + raise ValueError(f"pass formats must project explicit target cells: {mutation_pass.get('name')}") + if expected_architectures != set(mutation_pass.get("architectures", [])): + raise ValueError(f"pass architectures must project explicit target cells: {mutation_pass.get('name')}") + evidence_targets = set(_cell_evidence(mutation_pass)) + if not evidence_targets.issubset(statuses): + raise ValueError(f"evidence cell is not a declared target for pass {mutation_pass.get('name')}") + return statuses + + def build_matrix(document: dict[str, Any]) -> dict[str, Any]: """Build one explicit cell for every pass, format, and architecture.""" formats = tuple(document.get("formats", {})) @@ -667,22 +771,35 @@ def build_matrix(document: dict[str, Any]) -> dict[str, Any]: stability = mutation_pass.get("stability") if isinstance(stability, str): stability_counts[stability] = stability_counts.get(stability, 0) + 1 - supported_formats = set(mutation_pass.get("formats", [])) - supported_architectures = set(mutation_pass.get("architectures", [])) - evidence_cells = {(cell["format"], cell["architecture"]) for cell in mutation_pass.get("evidence_cells", [])} + target_statuses = _target_support_statuses( + mutation_pass, + formats, + architectures, + official_format, + official_architecture, + ) + evidence_cells = _cell_evidence(mutation_pass) for binary_format in formats: for architecture in architectures: target = (binary_format, architecture) - declared_support = binary_format in supported_formats and architecture in supported_architectures - evidenced = target in evidence_cells + support_status = target_statuses.get(target) + declared_support = support_status is not None + cell_is_evidenced = target in evidence_cells + references = evidence_cells.get(target, []) + if cell_is_evidenced and not references: + is_official_target = binary_format == official_format and architecture == official_architecture + references = mutation_pass.get("evidence", []) if is_official_target else [] + if not references: + raise ValueError(f"preview evidence cell needs target-specific references: {name}: {target}") cells.append( { "pass": name, "format": binary_format, "architecture": architecture, "declared_support": declared_support, - "status": "evidenced" if evidenced else "not-supported", - "evidence": mutation_pass.get("evidence", []) if evidenced else [], + "support_status": support_status, + "status": "evidenced" if cell_is_evidenced else "not-supported", + "evidence": references if cell_is_evidenced else [], } ) evidenced = sum(1 for cell in cells if cell["status"] == "evidenced") @@ -718,6 +835,12 @@ def build_matrix(document: dict[str, Any]) -> dict[str, Any]: maturity_gap_passes = _maturity_gap_passes(document.get("maturity")) maturity_gaps_by_pass = _maturity_gaps_by_pass(document.get("maturity")) native_evidence_gap_passes = _native_evidence_gap_passes(document.get("maturity")) + target_support_status_counts: dict[str, int] = {} + for cell in cells: + support_status = cell["support_status"] + if isinstance(support_status, str): + target_support_status_counts[support_status] = target_support_status_counts.get(support_status, 0) + 1 + declared_support_cells = sum(1 for cell in cells if cell["declared_support"]) non_official_gap_targets = _non_official_gap_targets(cells, official_format, official_architecture) parity_gap_scope = _parity_gap_scope(formats, architectures, official_format, official_architecture) vm_semantic_gap_scope = _vm_semantic_gap_scope(document) @@ -732,6 +855,9 @@ def build_matrix(document: dict[str, Any]) -> dict[str, Any]: }, "cell_count": len(cells), "summary": { + "declared_support_cells": declared_support_cells, + "unclaimed_cells": len(cells) - declared_support_cells, + "target_support_status_counts": dict(sorted(target_support_status_counts.items())), "evidenced_cells": evidenced, "not_supported_cells": len(cells) - evidenced, "official_evidenced_cells": official_evidenced, @@ -785,6 +911,7 @@ def build_matrix(document: dict[str, Any]) -> dict[str, Any]: "vm_semantic_blocker_totals": _vm_semantic_blocker_totals(vm_semantic_gap_scope), "vm_resistance_gap_scope": list(_VM_RESISTANCE_GAP_SCOPE), "vm_resistance_gap_evidence": _vm_resistance_gap_evidence(), + "vm_resistance_resolved_evidence": _vm_resistance_resolved_evidence(), "vm_resistance_evidence_blockers": _vm_resistance_evidence_blockers(), "vm_resistance_blocker_totals": _vm_resistance_blocker_totals(), }, diff --git a/tests/_doubles/relocation_order_rng.py b/tests/_doubles/relocation_order_rng.py new file mode 100644 index 00000000..2b9ad458 --- /dev/null +++ b/tests/_doubles/relocation_order_rng.py @@ -0,0 +1,5 @@ +class ReverseShuffle: + """Deterministically reverse a shuffle input for relocation tests.""" + + def shuffle[T](self, values: list[T]) -> None: + values.reverse() diff --git a/tests/integration/cli_test_support.py b/tests/integration/cli_test_support.py new file mode 100644 index 00000000..a0e8037a --- /dev/null +++ b/tests/integration/cli_test_support.py @@ -0,0 +1,55 @@ +"""Shared real-binary fixtures for CLI integration tests.""" + +import importlib.util +from pathlib import Path + +import pytest + +from r2morph.mutations.base import MutationPass +from tests.utils.assertions import expect + + +class CLIIntegrationBase: + """Provide the installed CLI prerequisites and ELF test fixture.""" + + @pytest.fixture(autouse=True) + def _require_r2pipe(self): + if importlib.util.find_spec("r2pipe") is None: + pytest.skip("r2pipe not installed") + if importlib.util.find_spec("yaml") is None: + pytest.skip("pyyaml not installed") + + @pytest.fixture + def ls_elf(self): + return Path(__file__).parent.parent.parent / "fixtures" / "dataset" / "elf_x86_64" + + +class _ReportFixturePass(MutationPass): + """Record one real binary region for report integration tests.""" + + def __init__(self): + super().__init__("ReportFixture") + + def apply(self, binary): + functions = binary.get_functions() + expect(functions) + func_addr = functions[0].get("offset", functions[0].get("addr", 0)) + instructions = binary.get_function_disasm(func_addr) + expect(instructions) + insn = instructions[0] + address = insn.get("addr", 0) + size = insn.get("size", 1) + original = binary.read_bytes(address, size) + expect(binary.write_bytes(address, original)) + self._record_mutation( + function_address=func_addr, + start_address=address, + end_address=address + size - 1, + original_bytes=original, + mutated_bytes=original, + original_disasm=insn.get("disasm", ""), + mutated_disasm=insn.get("disasm", ""), + mutation_kind="report_fixture", + metadata={"integration": True}, + ) + return {"mutations_applied": 1} diff --git a/tests/integration/test_block_reordering_pe_real.py b/tests/integration/test_block_reordering_pe_real.py new file mode 100644 index 00000000..a8d12d60 --- /dev/null +++ b/tests/integration/test_block_reordering_pe_real.py @@ -0,0 +1,81 @@ +"""Native execution regression for PE block reordering.""" + +import platform +import shutil +from pathlib import Path +from typing import Any + +import pytest + +from r2morph.core.binary import Binary +from r2morph.mutations.block_reordering import BlockReorderingPass +from r2morph.platform.pe_handler import PEHandler +from tests.utils.assertions import expect +from tests.utils.process import run_command + +_INPUTS = (-9, -2, -1, 0, 2, 5, 10, 18, 21) + + +def _pe_toolchain() -> tuple[str, str | None]: + compiler = shutil.which("x86_64-w64-mingw32-gcc") + is_windows = platform.system() == "Windows" + runtime = None if is_windows else shutil.which("wine") + if compiler is None or (not is_windows and runtime is None): + pytest.skip("a PE compiler and a native Windows or Wine runtime are required") + return compiler, runtime + + +def _build_pe_binary(directory: Path, compiler: str) -> Path: + source = directory / "block_reordering.c" + source.write_text( + "#include \n" + "#include \n" + "__attribute__((noinline)) int branchy(int value) {\n" + " int result;\n" + " if (value < 0) result = value * 3 + 7;\n" + " else result = value / 2 - 5;\n" + " if (result == 4) result += 11;\n" + " else result -= 3;\n" + " return result;\n" + "}\n" + "int main(int argc, char **argv) {\n" + " if (argc != 10) return 125;\n" + " for (int index = 1; index < argc; ++index) {\n" + ' printf("%d\\n", branchy((int)strtol(argv[index], 0, 10)));\n' + " }\n" + " return 0;\n" + "}\n", + encoding="ascii", + ) + binary = directory / "block_reordering.exe" + run_command([compiler, "-O0", "-fno-inline", "-o", str(binary), str(source)], check=True) + return binary + + +def _run_pe(binary: Path, runtime: str | None) -> Any: + arguments = [str(value) for value in _INPUTS] + command = [str(binary), *arguments] if runtime is None else [runtime, str(binary), *arguments] + # Wine combines host diagnostics with stderr; compare the child's exit and stdout. + return run_command(command, text=True, timeout=30) + + +def test_pe_x86_64_block_reordering_preserves_native_observables(tmp_path: Path) -> None: + compiler, runtime = _pe_toolchain() + binary_path = _build_pe_binary(tmp_path, compiler) + original = _run_pe(binary_path, runtime) + expect(len(set(original.stdout.splitlines())) > 1, "PE fixture inputs did not exercise distinct outcomes") + + with Binary(binary_path, writable=True) as binary: + binary.analyze("aa") + result = BlockReorderingPass({"probability": 1.0, "max_functions": 100, "seed": 1}).apply(binary) + + handler = PEHandler(binary_path) + expect(result["mutations_applied"] > 0, f"PE x86-64 block reordering made no change: {result!r}") + expect(handler.fix_checksum(), "failed to repair the PE checksum after block reordering") + expect(handler.validate_integrity()[0], "PE integrity validation failed after block reordering") + mutated = _run_pe(binary_path, runtime) + + expect( + (mutated.returncode, mutated.stdout) == (original.returncode, original.stdout), + "PE x86-64 block reordering changed return codes or stdout across the input set", + ) diff --git a/tests/integration/test_block_reordering_support_contract.py b/tests/integration/test_block_reordering_support_contract.py new file mode 100644 index 00000000..1244b466 --- /dev/null +++ b/tests/integration/test_block_reordering_support_contract.py @@ -0,0 +1,20 @@ +"""Regression coverage for the verified target boundary of block reordering.""" + +from r2morph.mutations.block_reordering import BlockReorderingPass +from tests.utils.assertions import expect + + +def test_block_reordering_keeps_non_elf_targets_as_previews() -> None: + support = BlockReorderingPass().get_support() + + expect( + support.formats == ("ELF", "Mach-O", "PE") + and support.architectures == ("x86_64",) + and support.stability == "stable" + and support.notes + == ( + "ELF x86-32 remains preview pending broader native corpus evidence.", + "Mach-O x86_64 remains preview pending broader native corpus evidence.", + "PE x86_64 remains preview pending broader native corpus evidence.", + ) + ) diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index b40201ad..14392056 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -4,19 +4,12 @@ import importlib import importlib.util -import json import sys -from pathlib import Path import pytest -from r2morph import MorphEngine -from r2morph.core.engine_run import EngineRunOptions -from r2morph.mutations.base import MutationPass -from r2morph.mutations.instruction_substitution import InstructionSubstitutionPass -from r2morph.mutations.register_substitution import RegisterSubstitutionPass -from tests.utils.assertions import expect, expect_all -from tests.utils.field_names import MUTATION_NAME_KEY, ONLY_FAILED_MUTATION_KEY, ONLY_MUTATION_KEY +from tests.integration.cli_test_support import CLIIntegrationBase +from tests.utils.assertions import expect from tests.utils.process import run_command _EXPECTED_FAILURES_COUNT_NOPINSERTION_NOT_REQUESTED_EXP_2 = 2 @@ -34,50 +27,9 @@ TYPER_AVAILABLE = False -class _ReportFixturePass(MutationPass): - def __init__(self): - super().__init__("ReportFixture") - - def apply(self, binary): - functions = binary.get_functions() - expect(functions) - func_addr = functions[0].get("offset", functions[0].get("addr", 0)) - instructions = binary.get_function_disasm(func_addr) - expect(instructions) - insn = instructions[0] - addr = insn.get("addr", 0) - size = insn.get("size", 1) - original = binary.read_bytes(addr, size) - expect(binary.write_bytes(addr, original)) - self._record_mutation( - function_address=func_addr, - start_address=addr, - end_address=addr + size - 1, - original_bytes=original, - mutated_bytes=original, - original_disasm=insn.get("disasm", ""), - mutated_disasm=insn.get("disasm", ""), - mutation_kind="report_fixture", - metadata={"integration": True}, - ) - return {"mutations_applied": 1} - - @pytest.mark.skipif(not TYPER_AVAILABLE, reason="typer not installed") -class TestCLI: - """Tests for r2morph CLI.""" - - @pytest.fixture(autouse=True) - def _require_r2pipe(self): - if importlib.util.find_spec("r2pipe") is None: - pytest.skip("r2pipe not installed") - if importlib.util.find_spec("yaml") is None: - pytest.skip("pyyaml not installed") - - @pytest.fixture - def ls_elf(self): - """Path to ls ELF binary.""" - return Path(__file__).parent.parent.parent / "fixtures" / "dataset" / "elf_x86_64" +class TestCLI(CLIIntegrationBase): + """CLI integration coverage for one cohesive command workflow.""" def test_cli_help(self): """Test CLI help command.""" @@ -306,2242 +258,3 @@ def test_cli_diff(self, ls_elf, tmp_path): ) expect(not (diff_result.returncode not in [0, 1])) - - def test_cli_report_filters_on_engine_generated_report(self, ls_elf, tmp_path): - """Test report filters against a real engine-generated symbolic report.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - report = tmp_path / "generated.report.json" - - with MorphEngine() as engine: - engine.load_binary(ls_elf).analyze() - engine.add_mutation(_ReportFixturePass()) - result = engine.run(EngineRunOptions(validation_mode="symbolic", report_path=report)) - - expect(report.exists()) - payload = json.loads(report.read_text(encoding="utf-8")) - expect(payload["mutations"]) - mutation = payload["mutations"][0] - symbolic_status = mutation["metadata"].get("symbolic_status") - expect(symbolic_status) - expect(not ("symbolic_issue_passes" not in payload["summary"])) - expect(not ("symbolic_coverage_by_pass" not in payload["summary"])) - expect(not ("symbolic_severity_by_pass" not in payload["summary"])) - expect(not ("symbolic_summary" not in payload["passes"]["ReportFixture"])) - expect(not ("severity" not in payload["passes"]["ReportFixture"]["symbolic_summary"])) - has_symbolic_issue = not mutation["metadata"].get("symbolic_observable_equivalent", False) and ( - mutation["metadata"].get("symbolic_observable_check_performed", False) - or symbolic_status - not in { - "real-binary-observables-match", - "shellcode-observables-match", - } - ) - if has_symbolic_issue: - expect(payload["summary"]["symbolic_issue_passes"]) - expect(payload["passes"]["ReportFixture"]["symbolic_summary"]["issues"]) - expect(payload["summary"]["symbolic_coverage_by_pass"]) - expect(not (payload["passes"]["ReportFixture"]["symbolic_summary"]["symbolic_requested"] < 1)) - expect(not (result["validation"]["symbolic"]["requested"] is not True)) - - summary_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--summary-only", - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - expect(summary_result.returncode == 0) - expect(not ("Symbolic Mutation Summary" not in summary_result.stdout)) - expect(not ("Severity Priority" not in summary_result.stdout)) - expect(not ("Pass Evidence" not in summary_result.stdout)) - if "RegisterSubstitution" in summary_result.stdout and "NopInsertion" in summary_result.stdout: - expect( - not (summary_result.stdout.index("RegisterSubstitution") >= summary_result.stdout.index("NopInsertion")) - ) - expect(not (has_symbolic_issue and "Passes With Symbolic Issues" not in summary_result.stdout)) - expect('"mutations"' not in summary_result.stdout) - - pass_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-pass", - "ReportFixture", - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - expect(pass_result.returncode == 0) - expect(not ("Pass Symbolic Summary" not in pass_result.stdout)) - expect(not ("Pass Evidence Summary" not in pass_result.stdout)) - expect(not ("severity=" not in pass_result.stdout)) - expect(not ('"pass_name": "ReportFixture"' not in pass_result.stdout)) - expect(not ('"report_filters": {' not in pass_result.stdout)) - expect(not ('"only_pass": "ReportFixture"' not in pass_result.stdout)) - - status_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-status", - symbolic_status, - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - expect(status_result.returncode == 0) - expect(not (f'"symbolic_status": "{symbolic_status}"' not in status_result.stdout)) - expect(not ('"report_filters": {' not in status_result.stdout)) - expect(not (f'"only_status": "{symbolic_status}"' not in status_result.stdout)) - - def test_cli_report_only_risky_passes_filters_real_risky_passes(self, deterministic_register_elf, tmp_path): - """Test `report --only-risky-passes` on a real report with symbolic mismatch evidence.""" - report = tmp_path / "register_risky.report.json" - filtered = tmp_path / "register_risky.filtered.json" - with MorphEngine(config={"seed": 1337}) as engine: - engine.load_binary(deterministic_register_elf).analyze() - engine.add_mutation( - RegisterSubstitutionPass( - config={ - "probability": 1.0, - "max_substitutions_per_function": 6, - "seed": 1337, - } - ) - ) - engine.run(EngineRunOptions(validation_mode="symbolic", seed=1337, report_path=report)) - - expect(report.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-risky-passes", - "--summary-only", - "--require-results", - "--output", - str(filtered), - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("Risky Pass Filter" not in report_result.stdout)) - expect(not ("RegisterSubstitution" not in report_result.stdout)) - - filtered_payload = json.loads(filtered.read_text(encoding="utf-8")) - expect(not (filtered_payload["report_filters"]["only_risky_passes"] is not True)) - expect(not ("RegisterSubstitution" not in filtered_payload["filtered_summary"]["risky_passes"])) - expect(not ("RegisterSubstitution" not in filtered_payload["filtered_summary"]["pass_risk_buckets"]["risky"])) - expect( - not ("RegisterSubstitution" not in filtered_payload["filtered_summary"]["pass_risk_buckets"]["symbolic"]) - ) - expect(filtered_payload["filtered_summary"]["pass_evidence"][0][MUTATION_NAME_KEY] == "RegisterSubstitution") - - def test_cli_report_only_symbolic_risk_filters_real_symbolic_risk(self, deterministic_register_elf, tmp_path): - """Test `report --only-symbolic-risk` on a real report with symbolic mismatch evidence.""" - report = tmp_path / "register_symbolic_risk.report.json" - filtered = tmp_path / "register_symbolic_risk.filtered.json" - - with MorphEngine(config={"seed": 1337}) as engine: - engine.load_binary(deterministic_register_elf).analyze() - engine.add_mutation( - RegisterSubstitutionPass( - config={ - "probability": 1.0, - "max_substitutions_per_function": 6, - "seed": 1337, - } - ) - ) - engine.run(EngineRunOptions(validation_mode="symbolic", seed=1337, report_path=report)) - - expect(report.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-symbolic-risk", - "--summary-only", - "--require-results", - "--output", - str(filtered), - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("Symbolic Risk Filter" not in report_result.stdout)) - expect(not ("RegisterSubstitution" not in report_result.stdout)) - - filtered_payload = json.loads(filtered.read_text(encoding="utf-8")) - expect(not (filtered_payload["report_filters"]["only_symbolic_risk"] is not True)) - expect(not ("RegisterSubstitution" not in filtered_payload["filtered_summary"]["symbolic_risk_passes"])) - expect(filtered_payload["filtered_summary"]["pass_evidence"][0][MUTATION_NAME_KEY] == "RegisterSubstitution") - - def test_cli_report_only_clean_passes_filters_real_clean_passes(self, ls_elf, tmp_path): - """Test `report --only-clean-passes` on a real report with clean symbolic evidence.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - report = tmp_path / "clean.report.json" - filtered = tmp_path / "clean.filtered.json" - - with MorphEngine() as engine: - engine.load_binary(ls_elf).analyze() - engine.add_mutation(_ReportFixturePass()) - engine.run(EngineRunOptions(validation_mode="off", report_path=report)) - - expect(report.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-clean-passes", - "--summary-only", - "--require-results", - "--output", - str(filtered), - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("Clean Pass Filter" not in report_result.stdout)) - expect(not ("ReportFixture" not in report_result.stdout)) - - filtered_payload = json.loads(filtered.read_text(encoding="utf-8")) - expect(not (filtered_payload["report_filters"]["only_clean_passes"] is not True)) - expect(not ("ReportFixture" not in filtered_payload["filtered_summary"]["clean_passes"])) - expect(not ("ReportFixture" not in filtered_payload["filtered_summary"]["pass_risk_buckets"]["clean"])) - expect(filtered_payload["filtered_summary"]["pass_evidence"][0][MUTATION_NAME_KEY] == "ReportFixture") - - def test_engine_generated_report_persists_pass_buckets(self, deterministic_register_elf, tmp_path): - """Test engine-generated reports persist risk/coverage buckets in summary.""" - report = tmp_path / "engine_buckets.report.json" - - with MorphEngine(config={"seed": 1337}) as engine: - engine.load_binary(deterministic_register_elf).analyze() - engine.add_mutation( - RegisterSubstitutionPass( - config={ - "probability": 1.0, - "max_substitutions_per_function": 6, - "seed": 1337, - } - ) - ) - engine.run(EngineRunOptions(validation_mode="symbolic", seed=1337, report_path=report)) - - payload = json.loads(report.read_text(encoding="utf-8")) - expect_all( - not ("pass_risk_buckets" not in payload), - not ("pass_coverage_buckets" not in payload), - not ("pass_risk_buckets" not in payload["summary"]), - not ("pass_coverage_buckets" not in payload["summary"]), - not ("pass_capabilities" not in payload["summary"]), - not ("pass_evidence_map" not in payload["summary"]), - not ("pass_evidence_priority" not in payload["summary"]), - not ("pass_triage_rows" not in payload["summary"]), - not ("pass_triage_map" not in payload["summary"]), - not ("pass_evidence_compact" not in payload["summary"]), - not ("normalized_pass_results" not in payload["summary"]), - not ("report_views" not in payload["summary"]), - not ("schema_version" not in payload), - not ("schema_version" not in payload["summary"]), - not ("validation_adjustment_rows" not in payload["summary"]), - not ("pass_capability_summary" not in payload["summary"]), - not ("pass_capability_summary_map" not in payload["summary"]), - not ("validation_role_rows" not in payload["summary"]), - not ("validation_role_map" not in payload["summary"]), - not ("validation_adjustments" not in payload["summary"]), - not ("symbolic_issue_map" not in payload["summary"]), - not ("symbolic_coverage_map" not in payload["summary"]), - not ("symbolic_severity_map" not in payload["summary"]), - not ("symbolic_status_counts" not in payload["summary"]), - not ("symbolic_status_rows" not in payload["summary"]), - not ("symbolic_status_map" not in payload["summary"]), - not ("symbolic_overview" not in payload["summary"]), - not ("observable_mismatch_by_pass" not in payload["summary"]), - not ("observable_mismatch_map" not in payload["summary"]), - not ("observable_mismatch_priority" not in payload["summary"]), - not ("discarded_mutation_summary" not in payload["summary"]), - not ("discarded_mutation_priority" not in payload["summary"]), - not ("RegisterSubstitution" not in payload["summary"]["pass_risk_buckets"]["risky"]), - not ("RegisterSubstitution" not in payload["summary"]["pass_risk_buckets"]["symbolic"]), - not ("RegisterSubstitution" not in payload["summary"]["report_views"]["passes"]["risky"]), - payload["summary"]["report_views"]["general_passes"][0][MUTATION_NAME_KEY] == "RegisterSubstitution", - not ("region_evidence_count" not in payload["summary"]["report_views"]["general_passes"][0]), - payload["summary"]["report_views"]["general_summary"]["passes"] == ["RegisterSubstitution"], - payload["summary"]["report_views"]["pass_filter_views"]["only_risky_passes"] == ["RegisterSubstitution"], - isinstance(payload["summary"]["report_views"]["mismatch_view"], list), - isinstance(payload["summary"]["report_views"]["only_mismatches"], dict), - not ("summary" not in payload["summary"]["report_views"]["only_mismatches"]), - not ("compact_rows" not in payload["summary"]["report_views"]["only_mismatches"]), - ) - if payload["summary"]["report_views"]["only_mismatches"]["rows"]: - expect_all( - not ("role" not in payload["summary"]["report_views"]["only_mismatches"]["rows"][0]), - not ("symbolic_confidence" not in payload["summary"]["report_views"]["only_mismatches"]["rows"][0]), - ) - expect_all( - isinstance(payload["summary"]["report_views"]["discarded_view"], dict), - isinstance(payload["summary"]["report_views"]["only_failed_gates"], dict), - not ("severity_priority" not in payload["summary"]["report_views"]["only_failed_gates"]), - not ("grouped_by_pass" not in payload["summary"]["report_views"]["only_failed_gates"]), - not ("compact_rows" not in payload["summary"]["report_views"]["only_failed_gates"]), - not ("expected_severity_counts" not in payload["summary"]["report_views"]["only_failed_gates"]), - not ("failed" not in payload["summary"]["report_views"]["only_failed_gates"]), - not ("failure_count" not in payload["summary"]["report_views"]["only_failed_gates"]), - isinstance(payload["summary"]["report_views"]["validation_adjustments"], dict), - not ("summary" not in payload["summary"]["report_views"]["validation_adjustments"]), - not ("by_impact" not in payload["summary"]["report_views"]["discarded_view"]), - not ("compact_rows" not in payload["summary"]["report_views"]["discarded_view"]), - isinstance(payload["summary"]["validation_adjustment_rows"], list), - isinstance(payload["summary"]["report_views"][ONLY_MUTATION_KEY], dict), - not ("pass_region_evidence_map" not in payload["summary"]), - not ("RegisterSubstitution" not in payload["summary"]["pass_capabilities"]), - not ("RegisterSubstitution" not in payload["summary"]["pass_capability_summary_map"]), - not ("RegisterSubstitution" not in payload["summary"]["pass_triage_map"]), - payload["summary"]["normalized_pass_results"][0][MUTATION_NAME_KEY] == "RegisterSubstitution", - payload["summary"]["pass_evidence_map"]["RegisterSubstitution"][MUTATION_NAME_KEY] - == "RegisterSubstitution", - payload["summary"]["pass_evidence_priority"][0][MUTATION_NAME_KEY] == "RegisterSubstitution", - payload["summary"]["symbolic_issue_map"]["RegisterSubstitution"][MUTATION_NAME_KEY] - == "RegisterSubstitution", - payload["summary"]["symbolic_coverage_map"]["RegisterSubstitution"][MUTATION_NAME_KEY] - == "RegisterSubstitution", - payload["summary"]["symbolic_severity_map"]["RegisterSubstitution"][MUTATION_NAME_KEY] - == "RegisterSubstitution", - isinstance(payload["summary"]["observable_mismatch_by_pass"], list), - isinstance(payload["summary"]["observable_mismatch_map"], dict), - ) - - def test_cli_report_only_covered_passes_filters_real_covered_passes(self, deterministic_substitute_elf, tmp_path): - """Test `report --only-covered-passes` on a real report with symbolic coverage.""" - report = tmp_path / "covered.report.json" - filtered = tmp_path / "covered.filtered.json" - - with MorphEngine(config={"seed": 1337}) as engine: - engine.load_binary(deterministic_substitute_elf).analyze() - engine.add_mutation( - InstructionSubstitutionPass( - config={ - "probability": 1.0, - "max_substitutions_per_function": 8, - "strict_size": True, - "seed": 1337, - } - ) - ) - engine.run(EngineRunOptions(validation_mode="symbolic", seed=1337, report_path=report)) - - expect(report.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-covered-passes", - "--summary-only", - "--require-results", - "--output", - str(filtered), - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("Covered Pass Filter" not in report_result.stdout)) - expect(not ("InstructionSubstitution" not in report_result.stdout)) - - filtered_payload = json.loads(filtered.read_text(encoding="utf-8")) - expect(not (filtered_payload["report_filters"]["only_covered_passes"] is not True)) - expect(not ("InstructionSubstitution" not in filtered_payload["filtered_summary"]["covered_passes"])) - expect( - not ( - "InstructionSubstitution" - not in filtered_payload["filtered_summary"]["pass_coverage_buckets"]["covered"] - ) - ) - expect(filtered_payload["filtered_summary"]["pass_evidence"][0][MUTATION_NAME_KEY] == "InstructionSubstitution") - - def test_cli_report_only_uncovered_passes_filters_real_uncovered_passes(self, ls_elf, tmp_path): - """Test `report --only-uncovered-passes` on a real clean report without symbolic coverage.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - report = tmp_path / "uncovered.report.json" - filtered = tmp_path / "uncovered.filtered.json" - - with MorphEngine() as engine: - engine.load_binary(ls_elf).analyze() - engine.add_mutation(_ReportFixturePass()) - engine.run(EngineRunOptions(validation_mode="off", report_path=report)) - - expect(report.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-uncovered-passes", - "--summary-only", - "--require-results", - "--output", - str(filtered), - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("Uncovered Pass Filter" not in report_result.stdout)) - expect(not ("ReportFixture" not in report_result.stdout)) - - filtered_payload = json.loads(filtered.read_text(encoding="utf-8")) - expect(not (filtered_payload["report_filters"]["only_uncovered_passes"] is not True)) - expect(not ("ReportFixture" not in filtered_payload["filtered_summary"]["uncovered_passes"])) - expect(not ("ReportFixture" not in filtered_payload["filtered_summary"]["pass_coverage_buckets"]["uncovered"])) - expect(filtered_payload["filtered_summary"]["pass_evidence"][0][MUTATION_NAME_KEY] == "ReportFixture") - - def test_cli_mutate_generated_report_supports_report_filters(self, ls_elf, tmp_path): - """Test `mutate --report` output can be consumed by `report --only-*` end-to-end.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "cli_symbolic.bin" - report = tmp_path / "cli_symbolic.report.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "symbolic", - "--allow-limited-symbolic", - "--seed", - "1337", - "-m", - "nop", - "-m", - "substitute", - "-m", - "register", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 0) - expect(output_path.exists()) - expect(report.exists()) - - payload = json.loads(report.read_text(encoding="utf-8")) - if not payload["mutations"]: - # The minimal ELF fixture (1 function) legitimately yields no - # kept mutations for nop/substitute/register; there is then - # no report-filter input to exercise here. - pytest.skip("CLI mutate produced no kept mutations on the minimal fixture") - mutation = next( - (item for item in payload["mutations"] if item.get("metadata", {}).get("symbolic_status")), - None, - ) - if mutation is None: - pytest.skip("No symbolic mutation metadata produced by CLI mutate run") - - pass_name = mutation[MUTATION_NAME_KEY] - symbolic_status = mutation["metadata"]["symbolic_status"] - - summary_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--summary-only", - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - expect(summary_result.returncode == 0) - expect(not ("Symbolic Mutation Summary" not in summary_result.stdout)) - - pass_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-pass", - pass_name, - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - expect(pass_result.returncode == 0) - expect(not (f'"pass_name": "{pass_name}"' not in pass_result.stdout)) - expect(not (f'"only_pass": "{pass_name}"' not in pass_result.stdout)) - - status_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-status", - symbolic_status, - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - expect(status_result.returncode == 0) - expect(not (f'"symbolic_status": "{symbolic_status}"' not in status_result.stdout)) - expect(not (f'"only_status": "{symbolic_status}"' not in status_result.stdout)) - - def test_cli_report_can_export_filtered_json(self, ls_elf, tmp_path): - """Test `report --output` writes a filtered JSON artifact from a real CLI report.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "cli_export.bin" - report = tmp_path / "cli_export.report.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "symbolic", - "--allow-limited-symbolic", - "--seed", - "1337", - "-m", - "nop", - "-m", - "substitute", - "-m", - "register", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 0) - expect(report.exists()) - - payload = json.loads(report.read_text(encoding="utf-8")) - mutation = next( - (item for item in payload.get("mutations", []) if item.get("metadata", {}).get("symbolic_status")), - None, - ) - if mutation is None: - pytest.skip("No symbolic mutation metadata produced by CLI mutate run") - - pass_name = mutation[MUTATION_NAME_KEY] - symbolic_status = mutation["metadata"]["symbolic_status"] - filtered_output = tmp_path / "filtered.report.json" - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-pass", - pass_name, - "--only-status", - symbolic_status, - "--output", - str(filtered_output), - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - expect(report_result.returncode == 0) - expect(filtered_output.exists()) - expect(not ("Filtered report written:" not in report_result.stdout)) - - filtered_payload = json.loads(filtered_output.read_text(encoding="utf-8")) - expect(filtered_payload["report_filters"][ONLY_MUTATION_KEY] == pass_name) - expect(filtered_payload["report_filters"]["only_status"] == symbolic_status) - expect(filtered_payload["filtered_summary"]["mutations"] == len(filtered_payload["mutations"])) - expect(filtered_payload["filtered_summary"]["passes"] == [pass_name]) - expect( - filtered_payload["filtered_summary"]["symbolic_statuses"] - == {symbolic_status: len(filtered_payload["mutations"])} - ) - expect(filtered_payload["mutations"]) - expect(all(item[MUTATION_NAME_KEY] == pass_name for item in filtered_payload["mutations"])) - expect( - all( - item.get("metadata", {}).get("symbolic_status") == symbolic_status - for item in filtered_payload["mutations"] - ) - ) - - def test_cli_report_require_results_uses_exit_code_for_ci(self, ls_elf, tmp_path): - """Test `report --require-results` succeeds or fails based on real filtered output.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "cli_require.bin" - report = tmp_path / "cli_require.report.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "symbolic", - "--allow-limited-symbolic", - "--seed", - "1337", - "-m", - "nop", - "-m", - "substitute", - "-m", - "register", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 0) - payload = json.loads(report.read_text(encoding="utf-8")) - mutation = next( - (item for item in payload.get("mutations", []) if item.get("metadata", {}).get("symbolic_status")), - None, - ) - if mutation is None: - pytest.skip("No symbolic mutation metadata produced by CLI mutate run") - - pass_name = mutation[MUTATION_NAME_KEY] - - success_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-pass", - pass_name, - "--require-results", - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - expect(success_result.returncode == 0) - expect(not (f'"only_pass": "{pass_name}"' not in success_result.stdout)) - - empty_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-pass", - "DefinitelyMissingPass", - "--require-results", - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - expect(empty_result.returncode == 1) - expect(not ('"mutations": []' not in empty_result.stdout)) - - def test_cli_report_require_results_supports_min_severity(self, ls_elf, tmp_path): - """Test `report --require-results --min-severity` on a real generated report.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "cli_require_severity.bin" - report = tmp_path / "cli_require_severity.report.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "symbolic", - "--allow-limited-symbolic", - "--seed", - "1337", - "-m", - "nop", - "-m", - "substitute", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 0) - - strict_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--require-results", - "--min-severity", - "not-requested", - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - failing_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-pass", - "DefinitelyMissingPass", - "--require-results", - "--min-severity", - "mismatch", - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(strict_result.returncode == 0) - expect(failing_result.returncode == 1) - - def test_cli_mutate_min_severity_can_pass(self, ls_elf, tmp_path): - """`mutate --min-severity` should succeed when the final report meets the threshold.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "cli_mutate_min_severity_ok.bin" - report = tmp_path / "cli_mutate_min_severity_ok.report.json" - - result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "structural", - "--min-severity", - "not-requested", - "--seed", - "1337", - "-m", - "nop", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(result.returncode == 0) - expect(output_path.exists()) - expect(report.exists()) - expect(not ("Severity gate passed:" not in result.stdout)) - - payload = json.loads(report.read_text(encoding="utf-8")) - expect(payload["summary"]["validation_mode"] == "structural") - expect(payload["summary"]["symbolic_severity_by_pass"][0]["severity"] == "not-requested") - expect(payload["gate_evaluation"]["requested"]["min_severity"] == "not-requested") - expect(not (payload["gate_evaluation"]["results"]["min_severity_passed"] is not True)) - expect(not (payload["gate_evaluation"]["results"]["all_passed"] is not True)) - - def test_cli_mutate_min_severity_can_fail_without_losing_artifacts(self, ls_elf, tmp_path): - """`mutate --min-severity` should fail with code 1 but keep output/report artifacts.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "cli_mutate_min_severity_fail.bin" - report = tmp_path / "cli_mutate_min_severity_fail.report.json" - - result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "structural", - "--min-severity", - "clean", - "--seed", - "1337", - "-m", - "nop", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(result.returncode == 1) - expect(output_path.exists()) - expect(report.exists()) - expect(not ("Severity gate failed:" not in result.stdout)) - - payload = json.loads(report.read_text(encoding="utf-8")) - expect(payload["summary"]["validation_mode"] == "structural") - expect(payload["summary"]["symbolic_severity_by_pass"][0]["severity"] == "not-requested") - expect(payload["gate_evaluation"]["requested"]["min_severity"] == "clean") - expect(not (payload["gate_evaluation"]["results"]["min_severity_passed"] is not False)) - expect(not (payload["gate_evaluation"]["results"]["all_passed"] is not False)) - expect(not (payload["gate_failures"]["min_severity_failed"] is not True)) - expect(not (payload["summary"]["gate_failures"]["min_severity_failed"] is not True)) - - def test_cli_mutate_require_pass_severity_can_pass(self, ls_elf, tmp_path): - """`mutate --require-pass-severity` should succeed when the named pass meets the threshold.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "cli_mutate_pass_severity_ok.bin" - report = tmp_path / "cli_mutate_pass_severity_ok.report.json" - - result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "structural", - "--require-pass-severity", - "NopInsertion=not-requested", - "--seed", - "1337", - "-m", - "nop", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(result.returncode == 0) - expect(output_path.exists()) - expect(report.exists()) - expect(not ("Pass severity gate passed:" not in result.stdout)) - expect(not ("NopInsertion<=not-requested" not in result.stdout)) - - payload = json.loads(report.read_text(encoding="utf-8")) - expect(payload["passes"]["NopInsertion"]["symbolic_summary"]["severity"] == "not-requested") - expect( - payload["gate_evaluation"]["requested"]["require_pass_severity"] - == [{"pass_name": "NopInsertion", "max_severity": "not-requested"}] - ) - expect(not (payload["gate_evaluation"]["results"]["require_pass_severity_passed"] is not True)) - expect(payload["gate_evaluation"]["results"]["require_pass_severity_failures"] == []) - expect(not (payload["gate_evaluation"]["results"]["all_passed"] is not True)) - - def test_cli_mutate_require_pass_severity_can_fail_without_losing_artifacts(self, ls_elf, tmp_path): - """`mutate --require-pass-severity` should fail with code 1 but keep artifacts.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "cli_mutate_pass_severity_fail.bin" - report = tmp_path / "cli_mutate_pass_severity_fail.report.json" - - result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "structural", - "--require-pass-severity", - "NopInsertion=clean", - "--seed", - "1337", - "-m", - "nop", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(result.returncode == 1) - expect(output_path.exists()) - expect(report.exists()) - expect(not ("Pass severity gate failed:" not in result.stdout)) - expect(not ("NopInsertion=not-requested(expected <= clean)" not in result.stdout)) - - payload = json.loads(report.read_text(encoding="utf-8")) - expect(payload["passes"]["NopInsertion"]["symbolic_summary"]["severity"] == "not-requested") - expect( - payload["gate_evaluation"]["requested"]["require_pass_severity"] - == [{"pass_name": "NopInsertion", "max_severity": "clean"}] - ) - expect(not (payload["gate_evaluation"]["results"]["require_pass_severity_passed"] is not False)) - expect( - payload["gate_evaluation"]["results"]["require_pass_severity_failures"] - == ["NopInsertion=not-requested(expected <= clean)"] - ) - expect(not (payload["gate_evaluation"]["results"]["all_passed"] is not False)) - expect(payload["gate_failures"]["require_pass_severity_failure_count"] == 1) - expect( - payload["gate_failure_priority"] - == [ - { - "pass_name": "NopInsertion", - "failure_count": 1, - "strictest_expected_severity": "clean", - "failures": ["NopInsertion=not-requested(expected <= clean)"], - } - ] - ) - expect(payload["gate_failure_severity_priority"] == [{"severity": "clean", "failure_count": 1}]) - expect(payload["gate_failures"]["require_pass_severity_failures_by_expected_severity"] == {"clean": 1}) - expect(payload["summary"]["gate_failures"]["require_pass_severity_failure_count"] == 1) - expect(payload["summary"]["gate_failure_priority"] == payload["gate_failure_priority"]) - expect(payload["summary"]["gate_failure_severity_priority"] == payload["gate_failure_severity_priority"]) - - def test_cli_mutate_require_pass_severity_accepts_mutation_alias(self, ls_elf, tmp_path): - """Short mutation aliases should resolve to the concrete pass name.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "cli_mutate_pass_alias_ok.bin" - report = tmp_path / "cli_mutate_pass_alias_ok.report.json" - - result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "structural", - "--require-pass-severity", - "nop=not-requested", - "--seed", - "1337", - "-m", - "nop", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(result.returncode == 0) - expect(output_path.exists()) - expect(report.exists()) - expect(not ("Pass severity gate passed:" not in result.stdout)) - expect(not ("NopInsertion<=not-requested" not in result.stdout)) - - def test_cli_mutate_require_pass_severity_alias_can_fail(self, ls_elf, tmp_path): - """Short mutation aliases should produce the same failure semantics as pass names.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "cli_mutate_pass_alias_fail.bin" - report = tmp_path / "cli_mutate_pass_alias_fail.report.json" - - result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "structural", - "--require-pass-severity", - "nop=clean", - "--seed", - "1337", - "-m", - "nop", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(result.returncode == 1) - expect(output_path.exists()) - expect(report.exists()) - expect(not ("Pass severity gate failed:" not in result.stdout)) - expect(not ("NopInsertion=not-requested(expected <= clean)" not in result.stdout)) - - def test_cli_mutate_require_pass_severity_rejects_unknown_alias(self, ls_elf, tmp_path): - """Unknown aliases should fail fast with exit code 2 before mutating.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "cli_mutate_pass_alias_invalid.bin" - report = tmp_path / "cli_mutate_pass_alias_invalid.report.json" - - result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "structural", - "--require-pass-severity", - "definitely-missing=clean", - "--seed", - "1337", - "-m", - "nop", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(result.returncode == _EXPECTED_RESULT_RETURNCODE_2) - expect(not (output_path.exists())) - expect(not (report.exists())) - expect(not ("Invalid --require-pass-severity: definitely-missing=clean" not in result.stdout)) - - def test_cli_symbolic_blocks_limited_pass_without_override( - self, - deterministic_register_elf, - tmp_path, - ): - """Symbolic mode should fail fast for passes that declare limited symbolic support.""" - output_path = tmp_path / "register_blocked.bin" - - result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(deterministic_register_elf), - "-o", - str(output_path), - "--validation-mode", - "symbolic", - "--seed", - "1337", - "-m", - "register", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(result.returncode == _EXPECTED_RESULT_RETURNCODE_2_2) - expect(not ("symbolic validation is marked limited" not in result.stdout)) - expect(not ("--allow-limited-symbolic" not in result.stdout)) - expect(not (output_path.exists())) - - def test_cli_symbolic_allows_limited_pass_with_override( - self, - deterministic_register_elf, - tmp_path, - ): - """The explicit override should allow the limited symbolic pass to run.""" - output_path = tmp_path / "register_allowed.bin" - report_path = tmp_path / "register_allowed.report.json" - - result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(deterministic_register_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "symbolic", - "--allow-limited-symbolic", - "--seed", - "1337", - "-m", - "register", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(result.returncode == 0) - expect(not ("Limited symbolic coverage explicitly allowed" not in result.stdout)) - expect(output_path.exists()) - payload = json.loads(report_path.read_text(encoding="utf-8")) - expect( - not ( - payload["pass_support"]["RegisterSubstitution"]["validator_capabilities"]["symbolic"]["recommended"] - is not False - ) - ) - - def test_cli_symbolic_can_degrade_limited_pass_to_runtime( - self, - deterministic_register_elf, - tmp_path, - ): - """Limited symbolic support can degrade to runtime validation instead of blocking.""" - output_path = tmp_path / "register_runtime.bin" - report_path = tmp_path / "register_runtime.report.json" - - result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(deterministic_register_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "symbolic", - "--limited-symbolic-policy", - "degrade-runtime", - "--seed", - "1337", - "-m", - "register", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(result.returncode == 0) - expect(not ("Degrading validation mode from symbolic to runtime" not in result.stdout)) - expect(output_path.exists()) - payload = json.loads(report_path.read_text(encoding="utf-8")) - expect(payload["summary"]["requested_validation_mode"] == "symbolic") - expect(payload["summary"]["validation_mode"] == "runtime") - expect(payload["summary"]["degradation_roles"]["degradation-trigger"] == 1) - expect(payload["validation_policy"]["policy"] == "degrade-runtime") - expect(payload["validation_policy"]["reason"] == "limited-symbolic-support") - expect(payload["validation_policy"]["limited_passes"][0]["role"] == "degradation-trigger") - expect(not (payload["validation"]["runtime"]["passed"] not in {True, False})) - expect( - payload["passes"]["RegisterSubstitution"]["validation_context"]["requested_validation_mode"] == "symbolic" - ) - expect( - payload["passes"]["RegisterSubstitution"]["validation_context"]["effective_validation_mode"] == "runtime" - ) - expect(not (payload["passes"]["RegisterSubstitution"]["validation_context"]["degraded_execution"] is not True)) - expect( - not ( - payload["passes"]["RegisterSubstitution"]["validation_context"]["degradation_triggered_by_pass"] - is not True - ) - ) - expect(payload["passes"]["RegisterSubstitution"]["validation_context"]["role"] == "degradation-trigger") - - def test_cli_symbolic_can_degrade_limited_pass_to_structural( - self, - deterministic_register_elf, - tmp_path, - ): - """Limited symbolic support can degrade to structural validation instead of blocking.""" - output_path = tmp_path / "register_structural.bin" - report_path = tmp_path / "register_structural.report.json" - - result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(deterministic_register_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "symbolic", - "--limited-symbolic-policy", - "degrade-structural", - "--seed", - "1337", - "-m", - "register", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(result.returncode == 0) - expect(not ("Degrading validation mode from symbolic to structural" not in result.stdout)) - expect(output_path.exists()) - payload = json.loads(report_path.read_text(encoding="utf-8")) - expect(payload["summary"]["requested_validation_mode"] == "symbolic") - expect(payload["summary"]["validation_mode"] == "structural") - expect(payload["summary"]["degradation_roles"]["degradation-trigger"] == 1) - expect(payload["validation_policy"]["policy"] == "degrade-structural") - expect(payload["validation_policy"]["reason"] == "limited-symbolic-support") - expect(payload["validation_policy"]["limited_passes"][0]["role"] == "degradation-trigger") - expect("runtime" not in payload["validation"]) - expect(payload["passes"]["RegisterSubstitution"]["validation_context"]["role"] == "degradation-trigger") - - def test_cli_report_can_filter_degraded_validation_runs( - self, - deterministic_register_elf, - tmp_path, - ): - """Report can triage a real run where validation mode was degraded.""" - output_path = tmp_path / "register_degraded.bin" - report_path = tmp_path / "register_degraded.report.json" - filtered_path = tmp_path / "register_degraded.filtered.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(deterministic_register_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "symbolic", - "--limited-symbolic-policy", - "degrade-runtime", - "--seed", - "1337", - "-m", - "register", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 0) - expect(report_path.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-degraded", - "--summary-only", - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("Validation Mode Adjustment" not in report_result.stdout)) - expect(not ("requested=symbolic, effective=runtime" not in report_result.stdout)) - expect(not ("Degradation Roles" not in report_result.stdout)) - expect(not ("Degraded Severity Priority" not in report_result.stdout)) - expect(not ("degradation-trigger: 1" not in report_result.stdout)) - expect(not ("Pass Validation Context" not in report_result.stdout)) - expect(not ("RegisterSubstitution" not in report_result.stdout)) - - export_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-degraded", - "--output", - str(filtered_path), - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(export_result.returncode == 0) - filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) - expect(not (filtered_payload["report_filters"]["only_degraded"] is not True)) - expect(not (filtered_payload["filtered_summary"]["degraded_validation"] is not True)) - expect(filtered_payload["filtered_summary"]["requested_validation_mode"] == "symbolic") - expect(filtered_payload["filtered_summary"]["validation_mode"] == "runtime") - expect(filtered_payload["filtered_summary"]["degraded_passes"]) - expect(filtered_payload["filtered_summary"]["degraded_passes"][0][MUTATION_NAME_KEY] == "RegisterSubstitution") - expect( - filtered_payload["filtered_summary"]["symbolic_severity_by_pass"][0][MUTATION_NAME_KEY] - == "RegisterSubstitution" - ) - expect(filtered_payload["filtered_summary"]["degradation_roles"]["degradation-trigger"] == 1) - - mismatch_filtered_path = tmp_path / "mismatch-filtered.json" - mismatch_export_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-degraded", - "--only-mismatches", - "--output", - str(mismatch_filtered_path), - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(mismatch_export_result.returncode == 0) - expect(not ("Mismatch Degradation Context" not in mismatch_export_result.stdout)) - expect(not ("Mismatch Severity Priority" not in mismatch_export_result.stdout)) - expect(not ("requested=symbolic, effective=runtime" not in mismatch_export_result.stdout)) - expect(not ("trigger_passes=RegisterSubstitution" not in mismatch_export_result.stdout)) - mismatch_payload = json.loads(mismatch_filtered_path.read_text(encoding="utf-8")) - expect(not (mismatch_payload["report_filters"]["only_degraded"] is not True)) - expect(not (mismatch_payload["report_filters"]["only_mismatches"] is not True)) - expect(mismatch_payload["filtered_summary"]["requested_validation_mode"] == "symbolic") - expect(mismatch_payload["filtered_summary"]["validation_mode"] == "runtime") - expect(not (mismatch_payload["filtered_summary"]["degraded_validation"] is not True)) - expect(mismatch_payload["filtered_summary"]["degraded_passes"][0][MUTATION_NAME_KEY] == "RegisterSubstitution") - expect(mismatch_payload["filtered_summary"]["degradation_roles"]["degradation-trigger"] == 1) - expect( - mismatch_payload["filtered_summary"]["symbolic_severity_by_pass"][0][MUTATION_NAME_KEY] - == "RegisterSubstitution" - ) - - def test_cli_report_can_filter_failed_gates(self, ls_elf, tmp_path): - """Report can triage a real run where mutate finished with failed CLI gates.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "failed_gate.bin" - report_path = tmp_path / "failed_gate.report.json" - filtered_path = tmp_path / "failed_gate.filtered.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "structural", - "--min-severity", - "clean", - "--seed", - "1337", - "-m", - "nop", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 1) - expect(report_path.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--summary-only", - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("Gate Evaluation" not in report_result.stdout)) - expect(not ("Gate Failure Summary" not in report_result.stdout)) - expect(not ("all_passed=no" not in report_result.stdout)) - expect(not ("min_severity=clean, passed=no" not in report_result.stdout)) - expect(not ("min_severity_failed=yes, require_pass_failures=0" not in report_result.stdout)) - - export_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--output", - str(filtered_path), - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(export_result.returncode == 0) - filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) - expect(not (filtered_payload["report_filters"]["only_failed_gates"] is not True)) - expect(not (filtered_payload["filtered_summary"]["failed_gates"] is not True)) - expect(filtered_payload["filtered_summary"]["gate_evaluation"]["requested"]["min_severity"] == "clean") - expect(not (filtered_payload["filtered_summary"]["gate_evaluation"]["results"]["all_passed"] is not False)) - expect(not (filtered_payload["filtered_summary"]["gate_failures"]["min_severity_failed"] is not True)) - expect(filtered_payload["filtered_summary"]["gate_failures"]["require_pass_severity_failure_count"] == 0) - - require_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--require-results", - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(require_result.returncode == 0) - expect(not ("Gate Evaluation" not in require_result.stdout)) - - def test_cli_report_groups_failed_pass_severity_gates(self, ls_elf, tmp_path): - """Report groups failed per-pass severity rules for a real mutate report.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "failed_pass_gate.bin" - report_path = tmp_path / "failed_pass_gate.report.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "structural", - "--require-pass-severity", - "nop=clean", - "--seed", - "1337", - "-m", - "nop", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 1) - expect(report_path.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--summary-only", - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("Gate Failure Summary" not in report_result.stdout)) - expect(not ("require_pass_failures=1" not in report_result.stdout)) - expect(not ("expected_severity_priority=clean:1" not in report_result.stdout)) - expect(not ("Gate Failure By Pass" not in report_result.stdout)) - expect(not ("NopInsertion" not in report_result.stdout)) - expect(not ("NopInsertion=not-requested(expected <= clean)" not in report_result.stdout)) - - payload = json.loads(report_path.read_text(encoding="utf-8")) - expect( - payload["gate_evaluation"]["results"]["require_pass_severity_failures"] - == ["NopInsertion=not-requested(expected <= clean)"] - ) - - def test_cli_report_only_expected_severity_filters_real_failed_gates(self, ls_elf, tmp_path): - """Report filters real failed gate views by expected severity.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "only_expected_severity.bin" - report_path = tmp_path / "only_expected_severity.report.json" - filtered_path = tmp_path / "only_expected_severity.filtered.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "structural", - "--require-pass-severity", - "nop=clean", - "--require-pass-severity", - "substitute=bounded-only", - "--seed", - "1337", - "-m", - "nop", - "-m", - "substitute", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 1) - expect(report_path.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--only-expected-severity", - "clean", - "--summary-only", - "--output", - str(filtered_path), - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("expected_severity_counts=clean:1" not in report_result.stdout)) - filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) - expect(filtered_payload["report_filters"]["only_expected_severity"] == "clean") - expect( - filtered_payload["filtered_summary"]["gate_failures"]["require_pass_severity_failures_by_expected_severity"] - == {"clean": 1} - ) - expect( - filtered_payload["filtered_summary"]["gate_failure_priority"] - == [ - { - "pass_name": "NopInsertion", - "failure_count": 1, - "strictest_expected_severity": "clean", - "failures": ["NopInsertion=not-requested(expected <= clean)"], - } - ] - ) - - def test_cli_report_only_expected_severity_require_results_on_real_failed_gates(self, ls_elf, tmp_path): - """Require-results should respect filtered expected-severity gate views.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "only_expected_severity_require_results.bin" - report_path = tmp_path / "only_expected_severity_require_results.report.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "structural", - "--require-pass-severity", - "nop=clean", - "--require-pass-severity", - "substitute=bounded-only", - "--seed", - "1337", - "-m", - "nop", - "-m", - "substitute", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 1) - expect(report_path.exists()) - - success_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--only-expected-severity", - "clean", - "--require-results", - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - failure_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--only-expected-severity", - "mismatch", - "--require-results", - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(success_result.returncode == 0) - expect(not ("Gate Failure Summary" not in success_result.stdout)) - expect(failure_result.returncode == 1) - - def test_cli_report_only_pass_failure_filters_real_failed_gates(self, ls_elf, tmp_path): - """Report filters real failed gates to a single pass failure.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "only_pass_failure.bin" - report_path = tmp_path / "only_pass_failure.report.json" - filtered_path = tmp_path / "only_pass_failure.filtered.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "structural", - "--require-pass-severity", - "nop=clean", - "--require-pass-severity", - "substitute=bounded-only", - "--seed", - "1337", - "-m", - "nop", - "-m", - "substitute", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 1) - expect(report_path.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--only-pass-failure", - "NopInsertion", - "--summary-only", - "--output", - str(filtered_path), - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("expected_severity_counts=clean:1" not in report_result.stdout)) - filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) - expect(filtered_payload["report_filters"][ONLY_FAILED_MUTATION_KEY] == "NopInsertion") - expect( - filtered_payload["filtered_summary"]["gate_failures"]["require_pass_severity_failures_by_pass"] - == {"NopInsertion": ["NopInsertion=not-requested(expected <= clean)"]} - ) - expect( - filtered_payload["filtered_summary"]["gate_failure_priority"] - == [ - { - "pass_name": "NopInsertion", - "failure_count": 1, - "strictest_expected_severity": "clean", - "failures": ["NopInsertion=not-requested(expected <= clean)"], - } - ] - ) - - def test_cli_report_only_pass_failure_require_results_on_real_failed_gates(self, ls_elf, tmp_path): - """Require-results should respect filtered pass-specific gate views.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "only_pass_failure_require_results.bin" - report_path = tmp_path / "only_pass_failure_require_results.report.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "structural", - "--require-pass-severity", - "nop=clean", - "--require-pass-severity", - "substitute=bounded-only", - "--seed", - "1337", - "-m", - "nop", - "-m", - "substitute", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 1) - expect(report_path.exists()) - - success_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--only-pass-failure", - "NopInsertion", - "--require-results", - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - failure_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--only-pass-failure", - "RegisterSubstitution", - "--require-results", - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(success_result.returncode == 0) - expect(not ("Gate Failure Summary" not in success_result.stdout)) - expect(failure_result.returncode == 1) - - def test_cli_report_only_pass_failure_accepts_mutation_alias(self, ls_elf, tmp_path): - """Report accepts stable mutation aliases for pass-failure filtering.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "only_pass_failure_alias.bin" - report_path = tmp_path / "only_pass_failure_alias.report.json" - filtered_path = tmp_path / "only_pass_failure_alias.filtered.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "structural", - "--require-pass-severity", - "nop=clean", - "--seed", - "1337", - "-m", - "nop", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 1) - expect(report_path.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--only-pass-failure", - "nop", - "--summary-only", - "--output", - str(filtered_path), - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("Pass Failure Filter Resolution" not in report_result.stdout)) - expect(not ("nop -> NopInsertion" not in report_result.stdout)) - filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) - expect(filtered_payload["report_filters"][ONLY_FAILED_MUTATION_KEY] == "NopInsertion") - expect(filtered_payload["filtered_summary"][ONLY_FAILED_MUTATION_KEY] == "NopInsertion") - - def test_cli_report_only_pass_accepts_mutation_alias(self, ls_elf, tmp_path): - """Report accepts stable mutation aliases for pass filtering.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "only_pass_alias.bin" - report = tmp_path / "only_pass_alias.report.json" - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report), - "--validation-mode", - "structural", - "--seed", - "1337", - "-m", - "nop", - ], - capture_output=True, - text=True, - timeout=90, - ) - expect(mutate_result.returncode == 0) - expect(report.exists()) - - pass_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-pass", - "nop", - str(report), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(pass_result.returncode == 0) - expect(not ("Pass Filter Resolution" not in pass_result.stdout)) - expect(not ("nop -> NopInsertion" not in pass_result.stdout)) - expect(not ('"report_filters": {' not in pass_result.stdout)) - expect(not ('"only_pass": "NopInsertion"' not in pass_result.stdout)) - expect(not ('"pass_name": "NopInsertion"' not in pass_result.stdout)) - - def test_cli_report_orders_failed_pass_severity_gates_by_expected_severity(self, ls_elf, tmp_path): - """Report orders grouped pass failures by stricter expected severity first.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "failed_pass_gate_order.bin" - report_path = tmp_path / "failed_pass_gate_order.report.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "structural", - "--require-pass-severity", - "nop=clean", - "--require-pass-severity", - "substitute=bounded-only", - "--seed", - "1337", - "-m", - "nop", - "-m", - "substitute", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 1) - expect(report_path.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--summary-only", - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("expected_severity_counts=bounded-only:1, clean:1" not in report_result.stdout)) - section = report_result.stdout.split("Gate Failure By Pass", 1)[1] - expect(not ("count=1, strictest_expected=bounded-only" not in section)) - expect(not ("count=1, strictest_expected=clean" not in section)) - expect(not (section.index("InstructionSubstitution") >= section.index("NopInsertion"))) - - payload = json.loads(report_path.read_text(encoding="utf-8")) - failures = payload["gate_evaluation"]["results"]["require_pass_severity_failures"] - expect(not ("InstructionSubstitution=not-requested(expected <= bounded-only)" not in failures)) - expect(not ("NopInsertion=not-requested(expected <= clean)" not in failures)) - - def test_cli_report_breaks_same_severity_gate_ties_by_failure_count(self, ls_elf, tmp_path): - """Report orders same-severity gate failures by number of failures for the pass.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "failed_pass_gate_count_order.bin" - report_path = tmp_path / "failed_pass_gate_count_order.report.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "structural", - "--require-pass-severity", - "nop=clean", - "--require-pass-severity", - "nop=clean", - "--require-pass-severity", - "substitute=clean", - "--seed", - "1337", - "-m", - "nop", - "-m", - "substitute", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 1) - expect(report_path.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--summary-only", - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - expect(not ("expected_severity_counts=clean:3" not in report_result.stdout)) - section = report_result.stdout.split("Gate Failure By Pass", 1)[1] - expect(not ("count=2, strictest_expected=clean" not in section)) - expect(not (section.index("NopInsertion") >= section.index("InstructionSubstitution"))) - - payload = json.loads(report_path.read_text(encoding="utf-8")) - failures = payload["gate_evaluation"]["results"]["require_pass_severity_failures"] - expect( - failures.count("NopInsertion=not-requested(expected <= clean)") - == _EXPECTED_FAILURES_COUNT_NOPINSERTION_NOT_REQUESTED_EXP_2 - ) - expect(not ("InstructionSubstitution=not-requested(expected <= clean)" not in failures)) - - def test_cli_report_exports_gate_failure_priority_for_real_failed_gates(self, ls_elf, tmp_path): - """Filtered report JSON preserves ordered gate failure priority for real runs.""" - if not ls_elf.exists(): - pytest.skip("ELF binary not available") - - output_path = tmp_path / "failed_pass_gate_priority.bin" - report_path = tmp_path / "failed_pass_gate_priority.report.json" - filtered_path = tmp_path / "failed_pass_gate_priority.filtered.json" - - mutate_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "mutate", - str(ls_elf), - "-o", - str(output_path), - "--report", - str(report_path), - "--validation-mode", - "structural", - "--require-pass-severity", - "nop=clean", - "--require-pass-severity", - "nop=clean", - "--require-pass-severity", - "substitute=clean", - "--seed", - "1337", - "-m", - "nop", - "-m", - "substitute", - ], - capture_output=True, - text=True, - timeout=90, - ) - - expect(mutate_result.returncode == 1) - expect(report_path.exists()) - - report_result = run_command( - [ - sys.executable, - "-m", - "r2morph.cli", - "report", - "--only-failed-gates", - "--summary-only", - "--output", - str(filtered_path), - str(report_path), - ], - capture_output=True, - text=True, - timeout=30, - ) - - expect(report_result.returncode == 0) - filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) - priority = filtered_payload["filtered_summary"]["gate_failure_priority"] - severity_counts = filtered_payload["filtered_summary"]["gate_failures"][ - "require_pass_severity_failures_by_expected_severity" - ] - expect([row[MUTATION_NAME_KEY] for row in priority] == ["NopInsertion", "InstructionSubstitution"]) - expect(priority[0]["failure_count"] == _EXPECTED_PRIORITY_0_FAILURE_COUNT_2) - expect(priority[0]["strictest_expected_severity"] == "clean") - expect(severity_counts == {"clean": 3}) diff --git a/tests/integration/test_cli_app_routes.py b/tests/integration/test_cli_app_routes.py index 59bad446..6475b81d 100644 --- a/tests/integration/test_cli_app_routes.py +++ b/tests/integration/test_cli_app_routes.py @@ -1,3 +1,4 @@ +import json from pathlib import Path from click.testing import CliRunner @@ -24,10 +25,75 @@ def test_cli_help_and_simple_mode(tmp_path): expect(output_path.exists()) -def test_cli_direct_analyze_and_functions(): +def test_cli_direct_analyze_and_functions(capfd): binary_path = Path("fixtures/dataset/elf_x86_64") analyze_cmd(binary_path, verbose=False) functions_cmd(binary_path, limit=5, verbose=False) + events = capfd.readouterr().err + expect("operation=cli.analyze" in events and "operation=cli.functions" in events) + + +def test_cli_diff_logs_operation_completion(capfd): + binary_path = Path("fixtures/dataset/elf_x86_64") + result = runner.invoke(app_cmd, ["diff", str(binary_path), str(binary_path)]) + stderr = capfd.readouterr().err + + expect(result.exit_code == 0 and "operation=cli.diff" in stderr) + + +def test_cli_validate_logs_runtime_outcome(capfd): + binary_path = Path("fixtures/dataset/elf_x86_64") + result = runner.invoke( + app_cmd, + ["validate", str(binary_path), str(binary_path)], + env={"R2MORPH_LOG_FORMAT": "json"}, + ) + stderr = capfd.readouterr().err + validation_result = json.loads(result.output) + outcome_event = next( + json.loads(line) for line in stderr.splitlines() if '"event_name":"cli.validate.result"' in line + ) + completion_event = next( + json.loads(line) for line in stderr.splitlines() if '"event_name":"cli.validate.completed"' in line + ) + + expect( + result.exit_code == (0 if validation_result["passed"] else 1) + and outcome_event["status"] == ("passed" if validation_result["passed"] else "failed") + and outcome_event["outcome"] == ("success" if validation_result["passed"] else "failure") + and outcome_event["correlation_id"] == completion_event["correlation_id"] + ) + + +def test_cli_enhanced_analysis_reuses_initialization_correlation_id(capfd): + binary_path = Path("fixtures/dataset/elf_x86_64") + result = runner.invoke( + app_cmd, + ["experimental", "analyze-enhanced", str(binary_path), "--detect-only"], + env={"R2MORPH_LOG_FORMAT": "json"}, + ) + events = [json.loads(line) for line in capfd.readouterr().err.splitlines()] + initialization = next( + event for event in events if event.get("event_name") == "cli.enhanced_analysis.initialize.completed" + ) + analysis = next(event for event in events if event.get("event_name") == "analysis.enhanced.completed") + + expect(result.exit_code == 0 and initialization["correlation_id"] == analysis["correlation_id"]) + + +def test_cli_honors_environment_debug_log_level(capfd): + binary_path = Path("fixtures/dataset/elf_x86_64") + result = runner.invoke( + app_cmd, + ["experimental", "analyze-enhanced", str(binary_path), "--detect-only"], + env={"R2MORPH_LOG_FORMAT": "json", "R2MORPH_LOG_LEVEL": "DEBUG"}, + ) + events = [json.loads(line) for line in capfd.readouterr().err.splitlines()] + + expect( + result.exit_code == 0 + and any(event.get("event_name") == "logging.configured" and event["level"] == "DEBUG" for event in events) + ) def test_cli_direct_morph(tmp_path): diff --git a/tests/integration/test_cli_report_coverage.py b/tests/integration/test_cli_report_coverage.py new file mode 100644 index 00000000..33e28615 --- /dev/null +++ b/tests/integration/test_cli_report_coverage.py @@ -0,0 +1,234 @@ +""" +Integration tests for CLI. +""" + +import importlib +import importlib.util +import json +import sys + +import pytest + +from r2morph import MorphEngine +from r2morph.core.engine_run import EngineRunOptions +from r2morph.mutations.instruction_substitution import InstructionSubstitutionPass +from r2morph.mutations.register_substitution import RegisterSubstitutionPass +from tests.integration.cli_test_support import CLIIntegrationBase, _ReportFixturePass +from tests.utils.assertions import expect, expect_all +from tests.utils.field_names import MUTATION_NAME_KEY, ONLY_MUTATION_KEY +from tests.utils.process import run_command + +_EXPECTED_FAILURES_COUNT_NOPINSERTION_NOT_REQUESTED_EXP_2 = 2 +_EXPECTED_PRIORITY_0_FAILURE_COUNT_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2_2 = 2 + + +# Check if typer is available +try: + importlib_util = importlib.import_module("importlib.util") + + TYPER_AVAILABLE = importlib_util.find_spec("typer") is not None +except ImportError: + TYPER_AVAILABLE = False + + +@pytest.mark.skipif(not TYPER_AVAILABLE, reason="typer not installed") +class TestCLI(CLIIntegrationBase): + """CLI integration coverage for one cohesive command workflow.""" + + def test_engine_generated_report_persists_pass_buckets(self, deterministic_register_elf, tmp_path): + """Test engine-generated reports persist risk/coverage buckets in summary.""" + report = tmp_path / "engine_buckets.report.json" + + with MorphEngine(config={"seed": 1337}) as engine: + engine.load_binary(deterministic_register_elf).analyze() + engine.add_mutation( + RegisterSubstitutionPass( + config={ + "probability": 1.0, + "max_substitutions_per_function": 6, + "seed": 1337, + } + ) + ) + engine.run(EngineRunOptions(validation_mode="symbolic", seed=1337, report_path=report)) + + payload = json.loads(report.read_text(encoding="utf-8")) + expect_all( + not ("pass_risk_buckets" not in payload), + not ("pass_coverage_buckets" not in payload), + not ("pass_risk_buckets" not in payload["summary"]), + not ("pass_coverage_buckets" not in payload["summary"]), + not ("pass_capabilities" not in payload["summary"]), + not ("pass_evidence_map" not in payload["summary"]), + not ("pass_evidence_priority" not in payload["summary"]), + not ("pass_triage_rows" not in payload["summary"]), + not ("pass_triage_map" not in payload["summary"]), + not ("pass_evidence_compact" not in payload["summary"]), + not ("normalized_pass_results" not in payload["summary"]), + not ("report_views" not in payload["summary"]), + not ("schema_version" not in payload), + not ("schema_version" not in payload["summary"]), + not ("validation_adjustment_rows" not in payload["summary"]), + not ("pass_capability_summary" not in payload["summary"]), + not ("pass_capability_summary_map" not in payload["summary"]), + not ("validation_role_rows" not in payload["summary"]), + not ("validation_role_map" not in payload["summary"]), + not ("validation_adjustments" not in payload["summary"]), + not ("symbolic_issue_map" not in payload["summary"]), + not ("symbolic_coverage_map" not in payload["summary"]), + not ("symbolic_severity_map" not in payload["summary"]), + not ("symbolic_status_counts" not in payload["summary"]), + not ("symbolic_status_rows" not in payload["summary"]), + not ("symbolic_status_map" not in payload["summary"]), + not ("symbolic_overview" not in payload["summary"]), + not ("observable_mismatch_by_pass" not in payload["summary"]), + not ("observable_mismatch_map" not in payload["summary"]), + not ("observable_mismatch_priority" not in payload["summary"]), + not ("discarded_mutation_summary" not in payload["summary"]), + not ("discarded_mutation_priority" not in payload["summary"]), + not ("RegisterSubstitution" not in payload["summary"]["pass_risk_buckets"]["risky"]), + not ("RegisterSubstitution" not in payload["summary"]["pass_risk_buckets"]["symbolic"]), + not ("RegisterSubstitution" not in payload["summary"]["report_views"]["passes"]["risky"]), + payload["summary"]["report_views"]["general_passes"][0][MUTATION_NAME_KEY] == "RegisterSubstitution", + not ("region_evidence_count" not in payload["summary"]["report_views"]["general_passes"][0]), + payload["summary"]["report_views"]["general_summary"]["passes"] == ["RegisterSubstitution"], + payload["summary"]["report_views"]["pass_filter_views"]["only_risky_passes"] == ["RegisterSubstitution"], + isinstance(payload["summary"]["report_views"]["mismatch_view"], list), + isinstance(payload["summary"]["report_views"]["only_mismatches"], dict), + not ("summary" not in payload["summary"]["report_views"]["only_mismatches"]), + not ("compact_rows" not in payload["summary"]["report_views"]["only_mismatches"]), + ) + if payload["summary"]["report_views"]["only_mismatches"]["rows"]: + expect_all( + not ("role" not in payload["summary"]["report_views"]["only_mismatches"]["rows"][0]), + not ("symbolic_confidence" not in payload["summary"]["report_views"]["only_mismatches"]["rows"][0]), + ) + expect_all( + isinstance(payload["summary"]["report_views"]["discarded_view"], dict), + isinstance(payload["summary"]["report_views"]["only_failed_gates"], dict), + not ("severity_priority" not in payload["summary"]["report_views"]["only_failed_gates"]), + not ("grouped_by_pass" not in payload["summary"]["report_views"]["only_failed_gates"]), + not ("compact_rows" not in payload["summary"]["report_views"]["only_failed_gates"]), + not ("expected_severity_counts" not in payload["summary"]["report_views"]["only_failed_gates"]), + not ("failed" not in payload["summary"]["report_views"]["only_failed_gates"]), + not ("failure_count" not in payload["summary"]["report_views"]["only_failed_gates"]), + isinstance(payload["summary"]["report_views"]["validation_adjustments"], dict), + not ("summary" not in payload["summary"]["report_views"]["validation_adjustments"]), + not ("by_impact" not in payload["summary"]["report_views"]["discarded_view"]), + not ("compact_rows" not in payload["summary"]["report_views"]["discarded_view"]), + isinstance(payload["summary"]["validation_adjustment_rows"], list), + isinstance(payload["summary"]["report_views"][ONLY_MUTATION_KEY], dict), + not ("pass_region_evidence_map" not in payload["summary"]), + not ("RegisterSubstitution" not in payload["summary"]["pass_capabilities"]), + not ("RegisterSubstitution" not in payload["summary"]["pass_capability_summary_map"]), + not ("RegisterSubstitution" not in payload["summary"]["pass_triage_map"]), + payload["summary"]["normalized_pass_results"][0][MUTATION_NAME_KEY] == "RegisterSubstitution", + payload["summary"]["pass_evidence_map"]["RegisterSubstitution"][MUTATION_NAME_KEY] + == "RegisterSubstitution", + payload["summary"]["pass_evidence_priority"][0][MUTATION_NAME_KEY] == "RegisterSubstitution", + payload["summary"]["symbolic_issue_map"]["RegisterSubstitution"][MUTATION_NAME_KEY] + == "RegisterSubstitution", + payload["summary"]["symbolic_coverage_map"]["RegisterSubstitution"][MUTATION_NAME_KEY] + == "RegisterSubstitution", + payload["summary"]["symbolic_severity_map"]["RegisterSubstitution"][MUTATION_NAME_KEY] + == "RegisterSubstitution", + isinstance(payload["summary"]["observable_mismatch_by_pass"], list), + isinstance(payload["summary"]["observable_mismatch_map"], dict), + ) + + def test_cli_report_only_covered_passes_filters_real_coverage(self, deterministic_substitute_elf, tmp_path): + """Filter according to measured coverage, including bounded-only evidence.""" + report = tmp_path / "covered.report.json" + filtered = tmp_path / "covered.filtered.json" + + with MorphEngine(config={"seed": 1337}) as engine: + engine.load_binary(deterministic_substitute_elf).analyze() + engine.add_mutation( + InstructionSubstitutionPass( + config={ + "probability": 1.0, + "max_substitutions_per_function": 8, + "strict_size": True, + "seed": 1337, + } + ) + ) + engine.run(EngineRunOptions(validation_mode="symbolic", seed=1337, report_path=report)) + + expect(report.exists()) + source_payload = json.loads(report.read_text(encoding="utf-8")) + covered_passes = source_payload["summary"]["pass_coverage_buckets"]["covered"] + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-covered-passes", + "--summary-only", + "--require-results", + "--output", + str(filtered), + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == (0 if covered_passes else 1)) + expect("Covered Pass Filter" in report_result.stdout) + + filtered_payload = json.loads(filtered.read_text(encoding="utf-8")) + expect( + filtered_payload["report_filters"]["only_covered_passes"] is True + and filtered_payload["filtered_summary"]["covered_passes"] == covered_passes + and filtered_payload["filtered_summary"]["pass_coverage_buckets"]["covered"] == covered_passes + and [row[MUTATION_NAME_KEY] for row in filtered_payload["filtered_summary"]["pass_evidence"]] + == covered_passes + ) + + def test_cli_report_only_uncovered_passes_filters_real_uncovered_passes(self, ls_elf, tmp_path): + """Test `report --only-uncovered-passes` on a real clean report without symbolic coverage.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + report = tmp_path / "uncovered.report.json" + filtered = tmp_path / "uncovered.filtered.json" + + with MorphEngine() as engine: + engine.load_binary(ls_elf).analyze() + engine.add_mutation(_ReportFixturePass()) + engine.run(EngineRunOptions(validation_mode="off", report_path=report)) + + expect(report.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-uncovered-passes", + "--summary-only", + "--require-results", + "--output", + str(filtered), + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("Uncovered Pass Filter" not in report_result.stdout)) + expect(not ("ReportFixture" not in report_result.stdout)) + + filtered_payload = json.loads(filtered.read_text(encoding="utf-8")) + expect(not (filtered_payload["report_filters"]["only_uncovered_passes"] is not True)) + expect(not ("ReportFixture" not in filtered_payload["filtered_summary"]["uncovered_passes"])) + expect(not ("ReportFixture" not in filtered_payload["filtered_summary"]["pass_coverage_buckets"]["uncovered"])) + expect(filtered_payload["filtered_summary"]["pass_evidence"][0][MUTATION_NAME_KEY] == "ReportFixture") diff --git a/tests/integration/test_cli_report_export_cases.py b/tests/integration/test_cli_report_export_cases.py new file mode 100644 index 00000000..5d13c760 --- /dev/null +++ b/tests/integration/test_cli_report_export_cases.py @@ -0,0 +1,233 @@ +""" +Integration tests for CLI. +""" + +import importlib +import importlib.util +import json +import sys + +import pytest + +from tests.integration.cli_test_support import CLIIntegrationBase +from tests.utils.assertions import expect +from tests.utils.field_names import MUTATION_NAME_KEY, ONLY_MUTATION_KEY +from tests.utils.process import run_command + +_EXPECTED_FAILURES_COUNT_NOPINSERTION_NOT_REQUESTED_EXP_2 = 2 +_EXPECTED_PRIORITY_0_FAILURE_COUNT_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2_2 = 2 + + +# Check if typer is available +try: + importlib_util = importlib.import_module("importlib.util") + + TYPER_AVAILABLE = importlib_util.find_spec("typer") is not None +except ImportError: + TYPER_AVAILABLE = False + + +@pytest.mark.skipif(not TYPER_AVAILABLE, reason="typer not installed") +class TestCLI(CLIIntegrationBase): + """CLI integration coverage for one cohesive command workflow.""" + + def test_cli_mutate_generated_report_supports_report_filters(self, ls_elf, tmp_path): + """Test `mutate --report` output can be consumed by `report --only-*` end-to-end.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "cli_symbolic.bin" + report = tmp_path / "cli_symbolic.report.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "symbolic", + "--allow-limited-symbolic", + "--seed", + "1337", + "-m", + "nop", + "-m", + "substitute", + "-m", + "register", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 0) + expect(output_path.exists()) + expect(report.exists()) + + payload = json.loads(report.read_text(encoding="utf-8")) + if not payload["mutations"]: + # The minimal ELF fixture (1 function) legitimately yields no + # kept mutations for nop/substitute/register; there is then + # no report-filter input to exercise here. + pytest.skip("CLI mutate produced no kept mutations on the minimal fixture") + mutation = next( + (item for item in payload["mutations"] if item.get("metadata", {}).get("symbolic_status")), + None, + ) + if mutation is None: + pytest.skip("No symbolic mutation metadata produced by CLI mutate run") + + pass_name = mutation[MUTATION_NAME_KEY] + symbolic_status = mutation["metadata"]["symbolic_status"] + + summary_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--summary-only", + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + expect(summary_result.returncode == 0) + expect(not ("Symbolic Mutation Summary" not in summary_result.stdout)) + + pass_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-pass", + pass_name, + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + expect(pass_result.returncode == 0) + expect(not (f'"pass_name": "{pass_name}"' not in pass_result.stdout)) + expect(not (f'"only_pass": "{pass_name}"' not in pass_result.stdout)) + + status_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-status", + symbolic_status, + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + expect(status_result.returncode == 0) + expect(not (f'"symbolic_status": "{symbolic_status}"' not in status_result.stdout)) + expect(not (f'"only_status": "{symbolic_status}"' not in status_result.stdout)) + + def test_cli_report_can_export_filtered_json(self, ls_elf, tmp_path): + """Test `report --output` writes a filtered JSON artifact from a real CLI report.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "cli_export.bin" + report = tmp_path / "cli_export.report.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "symbolic", + "--allow-limited-symbolic", + "--seed", + "1337", + "-m", + "nop", + "-m", + "substitute", + "-m", + "register", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 0) + expect(report.exists()) + + payload = json.loads(report.read_text(encoding="utf-8")) + mutation = next( + (item for item in payload.get("mutations", []) if item.get("metadata", {}).get("symbolic_status")), + None, + ) + if mutation is None: + pytest.skip("No symbolic mutation metadata produced by CLI mutate run") + + pass_name = mutation[MUTATION_NAME_KEY] + symbolic_status = mutation["metadata"]["symbolic_status"] + filtered_output = tmp_path / "filtered.report.json" + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-pass", + pass_name, + "--only-status", + symbolic_status, + "--output", + str(filtered_output), + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + expect(report_result.returncode == 0) + expect(filtered_output.exists()) + expect(not ("Filtered report written:" not in report_result.stdout)) + + filtered_payload = json.loads(filtered_output.read_text(encoding="utf-8")) + expect(filtered_payload["report_filters"][ONLY_MUTATION_KEY] == pass_name) + expect(filtered_payload["report_filters"]["only_status"] == symbolic_status) + expect(filtered_payload["filtered_summary"]["mutations"] == len(filtered_payload["mutations"])) + expect(filtered_payload["filtered_summary"]["passes"] == [pass_name]) + expect( + filtered_payload["filtered_summary"]["symbolic_statuses"] + == {symbolic_status: len(filtered_payload["mutations"])} + ) + expect(filtered_payload["mutations"]) + expect(all(item[MUTATION_NAME_KEY] == pass_name for item in filtered_payload["mutations"])) + expect( + all( + item.get("metadata", {}).get("symbolic_status") == symbolic_status + for item in filtered_payload["mutations"] + ) + ) diff --git a/tests/integration/test_cli_report_gate_priority.py b/tests/integration/test_cli_report_gate_priority.py new file mode 100644 index 00000000..730aa43d --- /dev/null +++ b/tests/integration/test_cli_report_gate_priority.py @@ -0,0 +1,243 @@ +""" +Integration tests for CLI. +""" + +import importlib +import importlib.util +import json +import sys + +import pytest + +from tests.integration.cli_test_support import CLIIntegrationBase +from tests.utils.assertions import expect +from tests.utils.field_names import MUTATION_NAME_KEY +from tests.utils.process import run_command + +_EXPECTED_FAILURES_COUNT_NOPINSERTION_NOT_REQUESTED_EXP_2 = 2 +_EXPECTED_PRIORITY_0_FAILURE_COUNT_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2_2 = 2 + + +# Check if typer is available +try: + importlib_util = importlib.import_module("importlib.util") + + TYPER_AVAILABLE = importlib_util.find_spec("typer") is not None +except ImportError: + TYPER_AVAILABLE = False + + +@pytest.mark.skipif(not TYPER_AVAILABLE, reason="typer not installed") +class TestCLI(CLIIntegrationBase): + """CLI integration coverage for one cohesive command workflow.""" + + def test_cli_report_orders_failed_pass_severity_gates_by_expected_severity(self, ls_elf, tmp_path): + """Report orders grouped pass failures by stricter expected severity first.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "failed_pass_gate_order.bin" + report_path = tmp_path / "failed_pass_gate_order.report.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "structural", + "--require-pass-severity", + "nop=clean", + "--require-pass-severity", + "substitute=bounded-only", + "--seed", + "1337", + "-m", + "nop", + "-m", + "substitute", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 1) + expect(report_path.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--summary-only", + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("expected_severity_counts=bounded-only:1, clean:1" not in report_result.stdout)) + section = report_result.stdout.split("Gate Failure By Pass", 1)[1] + expect(not ("count=1, strictest_expected=bounded-only" not in section)) + expect(not ("count=1, strictest_expected=clean" not in section)) + expect(not (section.index("InstructionSubstitution") >= section.index("NopInsertion"))) + + payload = json.loads(report_path.read_text(encoding="utf-8")) + failures = payload["gate_evaluation"]["results"]["require_pass_severity_failures"] + expect(not ("InstructionSubstitution=not-requested(expected <= bounded-only)" not in failures)) + expect(not ("NopInsertion=not-requested(expected <= clean)" not in failures)) + + def test_cli_report_breaks_same_severity_gate_ties_by_failure_count(self, ls_elf, tmp_path): + """Report orders same-severity gate failures by number of failures for the pass.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "failed_pass_gate_count_order.bin" + report_path = tmp_path / "failed_pass_gate_count_order.report.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "structural", + "--require-pass-severity", + "nop=clean", + "--require-pass-severity", + "nop=clean", + "--require-pass-severity", + "substitute=clean", + "--seed", + "1337", + "-m", + "nop", + "-m", + "substitute", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 1) + expect(report_path.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--summary-only", + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("expected_severity_counts=clean:3" not in report_result.stdout)) + section = report_result.stdout.split("Gate Failure By Pass", 1)[1] + expect(not ("count=2, strictest_expected=clean" not in section)) + expect(not (section.index("NopInsertion") >= section.index("InstructionSubstitution"))) + + payload = json.loads(report_path.read_text(encoding="utf-8")) + failures = payload["gate_evaluation"]["results"]["require_pass_severity_failures"] + expect( + failures.count("NopInsertion=not-requested(expected <= clean)") + == _EXPECTED_FAILURES_COUNT_NOPINSERTION_NOT_REQUESTED_EXP_2 + ) + expect(not ("InstructionSubstitution=not-requested(expected <= clean)" not in failures)) + + def test_cli_report_exports_gate_failure_priority_for_real_failed_gates(self, ls_elf, tmp_path): + """Filtered report JSON preserves ordered gate failure priority for real runs.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "failed_pass_gate_priority.bin" + report_path = tmp_path / "failed_pass_gate_priority.report.json" + filtered_path = tmp_path / "failed_pass_gate_priority.filtered.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "structural", + "--require-pass-severity", + "nop=clean", + "--require-pass-severity", + "nop=clean", + "--require-pass-severity", + "substitute=clean", + "--seed", + "1337", + "-m", + "nop", + "-m", + "substitute", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 1) + expect(report_path.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--summary-only", + "--output", + str(filtered_path), + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) + priority = filtered_payload["filtered_summary"]["gate_failure_priority"] + severity_counts = filtered_payload["filtered_summary"]["gate_failures"][ + "require_pass_severity_failures_by_expected_severity" + ] + expect([row[MUTATION_NAME_KEY] for row in priority] == ["NopInsertion", "InstructionSubstitution"]) + expect(priority[0]["failure_count"] == _EXPECTED_PRIORITY_0_FAILURE_COUNT_2) + expect(priority[0]["strictest_expected_severity"] == "clean") + expect(severity_counts == {"clean": 3}) diff --git a/tests/integration/test_cli_report_gates.py b/tests/integration/test_cli_report_gates.py new file mode 100644 index 00000000..08bf0a0a --- /dev/null +++ b/tests/integration/test_cli_report_gates.py @@ -0,0 +1,486 @@ +""" +Integration tests for CLI. +""" + +import importlib +import importlib.util +import json +import sys + +import pytest + +from tests.integration.cli_test_support import CLIIntegrationBase +from tests.utils.assertions import expect +from tests.utils.field_names import MUTATION_NAME_KEY +from tests.utils.process import run_command + +_EXPECTED_FAILURES_COUNT_NOPINSERTION_NOT_REQUESTED_EXP_2 = 2 +_EXPECTED_PRIORITY_0_FAILURE_COUNT_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2_2 = 2 + + +# Check if typer is available +try: + importlib_util = importlib.import_module("importlib.util") + + TYPER_AVAILABLE = importlib_util.find_spec("typer") is not None +except ImportError: + TYPER_AVAILABLE = False + + +@pytest.mark.skipif(not TYPER_AVAILABLE, reason="typer not installed") +class TestCLI(CLIIntegrationBase): + """CLI integration coverage for one cohesive command workflow.""" + + def test_cli_report_can_filter_degraded_validation_runs( + self, + deterministic_register_elf, + tmp_path, + ): + """Report can triage a real run where validation mode was degraded.""" + output_path = tmp_path / "register_degraded.bin" + report_path = tmp_path / "register_degraded.report.json" + filtered_path = tmp_path / "register_degraded.filtered.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(deterministic_register_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "symbolic", + "--limited-symbolic-policy", + "degrade-runtime", + "--seed", + "1337", + "-m", + "register", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 0) + expect(report_path.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-degraded", + "--summary-only", + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("Validation Mode Adjustment" not in report_result.stdout)) + expect(not ("requested=symbolic, effective=runtime" not in report_result.stdout)) + expect(not ("Degradation Roles" not in report_result.stdout)) + expect(not ("Degraded Severity Priority" not in report_result.stdout)) + expect(not ("degradation-trigger: 1" not in report_result.stdout)) + expect(not ("Pass Validation Context" not in report_result.stdout)) + expect(not ("RegisterSubstitution" not in report_result.stdout)) + + export_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-degraded", + "--output", + str(filtered_path), + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(export_result.returncode == 0) + filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) + expect(not (filtered_payload["report_filters"]["only_degraded"] is not True)) + expect(not (filtered_payload["filtered_summary"]["degraded_validation"] is not True)) + expect(filtered_payload["filtered_summary"]["requested_validation_mode"] == "symbolic") + expect(filtered_payload["filtered_summary"]["validation_mode"] == "runtime") + expect(filtered_payload["filtered_summary"]["degraded_passes"]) + expect(filtered_payload["filtered_summary"]["degraded_passes"][0][MUTATION_NAME_KEY] == "RegisterSubstitution") + expect( + filtered_payload["filtered_summary"]["symbolic_severity_by_pass"][0][MUTATION_NAME_KEY] + == "RegisterSubstitution" + ) + expect(filtered_payload["filtered_summary"]["degradation_roles"]["degradation-trigger"] == 1) + + mismatch_filtered_path = tmp_path / "mismatch-filtered.json" + mismatch_export_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-degraded", + "--only-mismatches", + "--output", + str(mismatch_filtered_path), + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(mismatch_export_result.returncode == 0) + expect(not ("Mismatch Degradation Context" not in mismatch_export_result.stdout)) + expect(not ("Mismatch Severity Priority" not in mismatch_export_result.stdout)) + expect(not ("requested=symbolic, effective=runtime" not in mismatch_export_result.stdout)) + expect(not ("trigger_passes=RegisterSubstitution" not in mismatch_export_result.stdout)) + mismatch_payload = json.loads(mismatch_filtered_path.read_text(encoding="utf-8")) + expect(not (mismatch_payload["report_filters"]["only_degraded"] is not True)) + expect(not (mismatch_payload["report_filters"]["only_mismatches"] is not True)) + expect(mismatch_payload["filtered_summary"]["requested_validation_mode"] == "symbolic") + expect(mismatch_payload["filtered_summary"]["validation_mode"] == "runtime") + expect(not (mismatch_payload["filtered_summary"]["degraded_validation"] is not True)) + expect(mismatch_payload["filtered_summary"]["degraded_passes"][0][MUTATION_NAME_KEY] == "RegisterSubstitution") + expect(mismatch_payload["filtered_summary"]["degradation_roles"]["degradation-trigger"] == 1) + expect( + mismatch_payload["filtered_summary"]["symbolic_severity_by_pass"][0][MUTATION_NAME_KEY] + == "RegisterSubstitution" + ) + + def test_cli_report_can_filter_failed_gates(self, ls_elf, tmp_path): + """Report can triage a real run where mutate finished with failed CLI gates.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "failed_gate.bin" + report_path = tmp_path / "failed_gate.report.json" + filtered_path = tmp_path / "failed_gate.filtered.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "structural", + "--min-severity", + "clean", + "--seed", + "1337", + "-m", + "nop", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 1) + expect(report_path.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--summary-only", + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("Gate Evaluation" not in report_result.stdout)) + expect(not ("Gate Failure Summary" not in report_result.stdout)) + expect(not ("all_passed=no" not in report_result.stdout)) + expect(not ("min_severity=clean, passed=no" not in report_result.stdout)) + expect(not ("min_severity_failed=yes, require_pass_failures=0" not in report_result.stdout)) + + export_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--output", + str(filtered_path), + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(export_result.returncode == 0) + filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) + expect(not (filtered_payload["report_filters"]["only_failed_gates"] is not True)) + expect(not (filtered_payload["filtered_summary"]["failed_gates"] is not True)) + expect(filtered_payload["filtered_summary"]["gate_evaluation"]["requested"]["min_severity"] == "clean") + expect(not (filtered_payload["filtered_summary"]["gate_evaluation"]["results"]["all_passed"] is not False)) + expect(not (filtered_payload["filtered_summary"]["gate_failures"]["min_severity_failed"] is not True)) + expect(filtered_payload["filtered_summary"]["gate_failures"]["require_pass_severity_failure_count"] == 0) + + require_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--require-results", + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(require_result.returncode == 0) + expect(not ("Gate Evaluation" not in require_result.stdout)) + + def test_cli_report_groups_failed_pass_severity_gates(self, ls_elf, tmp_path): + """Report groups failed per-pass severity rules for a real mutate report.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "failed_pass_gate.bin" + report_path = tmp_path / "failed_pass_gate.report.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "structural", + "--require-pass-severity", + "nop=clean", + "--seed", + "1337", + "-m", + "nop", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 1) + expect(report_path.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--summary-only", + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("Gate Failure Summary" not in report_result.stdout)) + expect(not ("require_pass_failures=1" not in report_result.stdout)) + expect(not ("expected_severity_priority=clean:1" not in report_result.stdout)) + expect(not ("Gate Failure By Pass" not in report_result.stdout)) + expect(not ("NopInsertion" not in report_result.stdout)) + expect(not ("NopInsertion=not-requested(expected <= clean)" not in report_result.stdout)) + + payload = json.loads(report_path.read_text(encoding="utf-8")) + expect( + payload["gate_evaluation"]["results"]["require_pass_severity_failures"] + == ["NopInsertion=not-requested(expected <= clean)"] + ) + + def test_cli_report_only_expected_severity_filters_real_failed_gates(self, ls_elf, tmp_path): + """Report filters real failed gate views by expected severity.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "only_expected_severity.bin" + report_path = tmp_path / "only_expected_severity.report.json" + filtered_path = tmp_path / "only_expected_severity.filtered.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "structural", + "--require-pass-severity", + "nop=clean", + "--require-pass-severity", + "substitute=bounded-only", + "--seed", + "1337", + "-m", + "nop", + "-m", + "substitute", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 1) + expect(report_path.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--only-expected-severity", + "clean", + "--summary-only", + "--output", + str(filtered_path), + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("expected_severity_counts=clean:1" not in report_result.stdout)) + filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) + expect(filtered_payload["report_filters"]["only_expected_severity"] == "clean") + expect( + filtered_payload["filtered_summary"]["gate_failures"]["require_pass_severity_failures_by_expected_severity"] + == {"clean": 1} + ) + expect( + filtered_payload["filtered_summary"]["gate_failure_priority"] + == [ + { + "pass_name": "NopInsertion", + "failure_count": 1, + "strictest_expected_severity": "clean", + "failures": ["NopInsertion=not-requested(expected <= clean)"], + } + ] + ) + + def test_cli_report_only_expected_severity_require_results_on_real_failed_gates(self, ls_elf, tmp_path): + """Require-results should respect filtered expected-severity gate views.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "only_expected_severity_require_results.bin" + report_path = tmp_path / "only_expected_severity_require_results.report.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "structural", + "--require-pass-severity", + "nop=clean", + "--require-pass-severity", + "substitute=bounded-only", + "--seed", + "1337", + "-m", + "nop", + "-m", + "substitute", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 1) + expect(report_path.exists()) + + success_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--only-expected-severity", + "clean", + "--require-results", + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + failure_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--only-expected-severity", + "mismatch", + "--require-results", + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(success_result.returncode == 0) + expect(not ("Gate Failure Summary" not in success_result.stdout)) + expect(failure_result.returncode == 1) diff --git a/tests/integration/test_cli_report_output_policy.py b/tests/integration/test_cli_report_output_policy.py new file mode 100644 index 00000000..4828a116 --- /dev/null +++ b/tests/integration/test_cli_report_output_policy.py @@ -0,0 +1,330 @@ +""" +Integration tests for CLI. +""" + +import importlib +import importlib.util +import json +import sys + +import pytest + +from tests.integration.cli_test_support import CLIIntegrationBase +from tests.utils.assertions import expect +from tests.utils.field_names import MUTATION_NAME_KEY +from tests.utils.process import run_command + +_EXPECTED_FAILURES_COUNT_NOPINSERTION_NOT_REQUESTED_EXP_2 = 2 +_EXPECTED_PRIORITY_0_FAILURE_COUNT_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2_2 = 2 + + +# Check if typer is available +try: + importlib_util = importlib.import_module("importlib.util") + + TYPER_AVAILABLE = importlib_util.find_spec("typer") is not None +except ImportError: + TYPER_AVAILABLE = False + + +@pytest.mark.skipif(not TYPER_AVAILABLE, reason="typer not installed") +class TestCLI(CLIIntegrationBase): + """CLI integration coverage for one cohesive command workflow.""" + + def test_cli_report_require_results_uses_exit_code_for_ci(self, ls_elf, tmp_path): + """Test `report --require-results` succeeds or fails based on real filtered output.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "cli_require.bin" + report = tmp_path / "cli_require.report.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "symbolic", + "--allow-limited-symbolic", + "--seed", + "1337", + "-m", + "nop", + "-m", + "substitute", + "-m", + "register", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 0) + payload = json.loads(report.read_text(encoding="utf-8")) + mutation = next( + (item for item in payload.get("mutations", []) if item.get("metadata", {}).get("symbolic_status")), + None, + ) + if mutation is None: + pytest.skip("No symbolic mutation metadata produced by CLI mutate run") + + pass_name = mutation[MUTATION_NAME_KEY] + + success_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-pass", + pass_name, + "--require-results", + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + expect(success_result.returncode == 0) + expect(not (f'"only_pass": "{pass_name}"' not in success_result.stdout)) + + empty_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-pass", + "DefinitelyMissingPass", + "--require-results", + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + expect(empty_result.returncode == 1) + expect(not ('"mutations": []' not in empty_result.stdout)) + + def test_cli_report_require_results_supports_min_severity(self, ls_elf, tmp_path): + """Test `report --require-results --min-severity` on a real generated report.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "cli_require_severity.bin" + report = tmp_path / "cli_require_severity.report.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "symbolic", + "--allow-limited-symbolic", + "--seed", + "1337", + "-m", + "nop", + "-m", + "substitute", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 0) + + strict_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--require-results", + "--min-severity", + "not-requested", + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + failing_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-pass", + "DefinitelyMissingPass", + "--require-results", + "--min-severity", + "mismatch", + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(strict_result.returncode == 0) + expect(failing_result.returncode == 1) + + def test_cli_mutate_min_severity_can_pass(self, ls_elf, tmp_path): + """`mutate --min-severity` should succeed when the final report meets the threshold.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "cli_mutate_min_severity_ok.bin" + report = tmp_path / "cli_mutate_min_severity_ok.report.json" + + result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "structural", + "--min-severity", + "not-requested", + "--seed", + "1337", + "-m", + "nop", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(result.returncode == 0) + expect(output_path.exists()) + expect(report.exists()) + expect(not ("Severity gate passed:" not in result.stdout)) + + payload = json.loads(report.read_text(encoding="utf-8")) + expect(payload["summary"]["validation_mode"] == "structural") + expect(payload["summary"]["symbolic_severity_by_pass"][0]["severity"] == "not-requested") + expect(payload["gate_evaluation"]["requested"]["min_severity"] == "not-requested") + expect(not (payload["gate_evaluation"]["results"]["min_severity_passed"] is not True)) + expect(not (payload["gate_evaluation"]["results"]["all_passed"] is not True)) + + def test_cli_mutate_min_severity_can_fail_without_losing_artifacts(self, ls_elf, tmp_path): + """`mutate --min-severity` should fail with code 1 but keep output/report artifacts.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "cli_mutate_min_severity_fail.bin" + report = tmp_path / "cli_mutate_min_severity_fail.report.json" + + result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "structural", + "--min-severity", + "clean", + "--seed", + "1337", + "-m", + "nop", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(result.returncode == 1) + expect(output_path.exists()) + expect(report.exists()) + expect(not ("Severity gate failed:" not in result.stdout)) + + payload = json.loads(report.read_text(encoding="utf-8")) + expect(payload["summary"]["validation_mode"] == "structural") + expect(payload["summary"]["symbolic_severity_by_pass"][0]["severity"] == "not-requested") + expect(payload["gate_evaluation"]["requested"]["min_severity"] == "clean") + expect(not (payload["gate_evaluation"]["results"]["min_severity_passed"] is not False)) + expect(not (payload["gate_evaluation"]["results"]["all_passed"] is not False)) + expect(not (payload["gate_failures"]["min_severity_failed"] is not True)) + expect(not (payload["summary"]["gate_failures"]["min_severity_failed"] is not True)) + + def test_cli_mutate_require_pass_severity_can_pass(self, ls_elf, tmp_path): + """`mutate --require-pass-severity` should succeed when the named pass meets the threshold.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "cli_mutate_pass_severity_ok.bin" + report = tmp_path / "cli_mutate_pass_severity_ok.report.json" + + result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "structural", + "--require-pass-severity", + "NopInsertion=not-requested", + "--seed", + "1337", + "-m", + "nop", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(result.returncode == 0) + expect(output_path.exists()) + expect(report.exists()) + expect(not ("Pass severity gate passed:" not in result.stdout)) + expect(not ("NopInsertion<=not-requested" not in result.stdout)) + + payload = json.loads(report.read_text(encoding="utf-8")) + expect(payload["passes"]["NopInsertion"]["symbolic_summary"]["severity"] == "not-requested") + expect( + payload["gate_evaluation"]["requested"]["require_pass_severity"] + == [{"pass_name": "NopInsertion", "max_severity": "not-requested"}] + ) + expect(not (payload["gate_evaluation"]["results"]["require_pass_severity_passed"] is not True)) + expect(payload["gate_evaluation"]["results"]["require_pass_severity_failures"] == []) + expect(not (payload["gate_evaluation"]["results"]["all_passed"] is not True)) diff --git a/tests/integration/test_cli_report_pass_failures.py b/tests/integration/test_cli_report_pass_failures.py new file mode 100644 index 00000000..49cd49dd --- /dev/null +++ b/tests/integration/test_cli_report_pass_failures.py @@ -0,0 +1,308 @@ +""" +Integration tests for CLI. +""" + +import importlib +import importlib.util +import json +import sys + +import pytest + +from tests.integration.cli_test_support import CLIIntegrationBase +from tests.utils.assertions import expect +from tests.utils.field_names import ONLY_FAILED_MUTATION_KEY +from tests.utils.process import run_command + +_EXPECTED_FAILURES_COUNT_NOPINSERTION_NOT_REQUESTED_EXP_2 = 2 +_EXPECTED_PRIORITY_0_FAILURE_COUNT_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2_2 = 2 + + +# Check if typer is available +try: + importlib_util = importlib.import_module("importlib.util") + + TYPER_AVAILABLE = importlib_util.find_spec("typer") is not None +except ImportError: + TYPER_AVAILABLE = False + + +@pytest.mark.skipif(not TYPER_AVAILABLE, reason="typer not installed") +class TestCLI(CLIIntegrationBase): + """CLI integration coverage for one cohesive command workflow.""" + + def test_cli_report_only_pass_failure_filters_real_failed_gates(self, ls_elf, tmp_path): + """Report filters real failed gates to a single pass failure.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "only_pass_failure.bin" + report_path = tmp_path / "only_pass_failure.report.json" + filtered_path = tmp_path / "only_pass_failure.filtered.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "structural", + "--require-pass-severity", + "nop=clean", + "--require-pass-severity", + "substitute=bounded-only", + "--seed", + "1337", + "-m", + "nop", + "-m", + "substitute", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 1) + expect(report_path.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--only-pass-failure", + "NopInsertion", + "--summary-only", + "--output", + str(filtered_path), + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("expected_severity_counts=clean:1" not in report_result.stdout)) + filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) + expect(filtered_payload["report_filters"][ONLY_FAILED_MUTATION_KEY] == "NopInsertion") + expect( + filtered_payload["filtered_summary"]["gate_failures"]["require_pass_severity_failures_by_pass"] + == {"NopInsertion": ["NopInsertion=not-requested(expected <= clean)"]} + ) + expect( + filtered_payload["filtered_summary"]["gate_failure_priority"] + == [ + { + "pass_name": "NopInsertion", + "failure_count": 1, + "strictest_expected_severity": "clean", + "failures": ["NopInsertion=not-requested(expected <= clean)"], + } + ] + ) + + def test_cli_report_only_pass_failure_require_results_on_real_failed_gates(self, ls_elf, tmp_path): + """Require-results should respect filtered pass-specific gate views.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "only_pass_failure_require_results.bin" + report_path = tmp_path / "only_pass_failure_require_results.report.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "structural", + "--require-pass-severity", + "nop=clean", + "--require-pass-severity", + "substitute=bounded-only", + "--seed", + "1337", + "-m", + "nop", + "-m", + "substitute", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 1) + expect(report_path.exists()) + + success_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--only-pass-failure", + "NopInsertion", + "--require-results", + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + failure_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--only-pass-failure", + "RegisterSubstitution", + "--require-results", + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(success_result.returncode == 0) + expect(not ("Gate Failure Summary" not in success_result.stdout)) + expect(failure_result.returncode == 1) + + def test_cli_report_only_pass_failure_accepts_mutation_alias(self, ls_elf, tmp_path): + """Report accepts stable mutation aliases for pass-failure filtering.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "only_pass_failure_alias.bin" + report_path = tmp_path / "only_pass_failure_alias.report.json" + filtered_path = tmp_path / "only_pass_failure_alias.filtered.json" + + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "structural", + "--require-pass-severity", + "nop=clean", + "--seed", + "1337", + "-m", + "nop", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(mutate_result.returncode == 1) + expect(report_path.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-failed-gates", + "--only-pass-failure", + "nop", + "--summary-only", + "--output", + str(filtered_path), + str(report_path), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("Pass Failure Filter Resolution" not in report_result.stdout)) + expect(not ("nop -> NopInsertion" not in report_result.stdout)) + filtered_payload = json.loads(filtered_path.read_text(encoding="utf-8")) + expect(filtered_payload["report_filters"][ONLY_FAILED_MUTATION_KEY] == "NopInsertion") + expect(filtered_payload["filtered_summary"][ONLY_FAILED_MUTATION_KEY] == "NopInsertion") + + def test_cli_report_only_pass_accepts_mutation_alias(self, ls_elf, tmp_path): + """Report accepts stable mutation aliases for pass filtering.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "only_pass_alias.bin" + report = tmp_path / "only_pass_alias.report.json" + mutate_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "structural", + "--seed", + "1337", + "-m", + "nop", + ], + capture_output=True, + text=True, + timeout=90, + ) + expect(mutate_result.returncode == 0) + expect(report.exists()) + + pass_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-pass", + "nop", + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(pass_result.returncode == 0) + expect(not ("Pass Filter Resolution" not in pass_result.stdout)) + expect(not ("nop -> NopInsertion" not in pass_result.stdout)) + expect(not ('"report_filters": {' not in pass_result.stdout)) + expect(not ('"only_pass": "NopInsertion"' not in pass_result.stdout)) + expect(not ('"pass_name": "NopInsertion"' not in pass_result.stdout)) diff --git a/tests/integration/test_cli_report_severity.py b/tests/integration/test_cli_report_severity.py new file mode 100644 index 00000000..83464e59 --- /dev/null +++ b/tests/integration/test_cli_report_severity.py @@ -0,0 +1,403 @@ +""" +Integration tests for CLI. +""" + +import importlib +import importlib.util +import json +import sys + +import pytest + +from tests.integration.cli_test_support import CLIIntegrationBase +from tests.utils.assertions import expect +from tests.utils.process import run_command + +_EXPECTED_FAILURES_COUNT_NOPINSERTION_NOT_REQUESTED_EXP_2 = 2 +_EXPECTED_PRIORITY_0_FAILURE_COUNT_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2_2 = 2 + + +# Check if typer is available +try: + importlib_util = importlib.import_module("importlib.util") + + TYPER_AVAILABLE = importlib_util.find_spec("typer") is not None +except ImportError: + TYPER_AVAILABLE = False + + +@pytest.mark.skipif(not TYPER_AVAILABLE, reason="typer not installed") +class TestCLI(CLIIntegrationBase): + """CLI integration coverage for one cohesive command workflow.""" + + def test_cli_mutate_require_pass_severity_can_fail_without_losing_artifacts(self, ls_elf, tmp_path): + """`mutate --require-pass-severity` should fail with code 1 but keep artifacts.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "cli_mutate_pass_severity_fail.bin" + report = tmp_path / "cli_mutate_pass_severity_fail.report.json" + + result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "structural", + "--require-pass-severity", + "NopInsertion=clean", + "--seed", + "1337", + "-m", + "nop", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(result.returncode == 1) + expect(output_path.exists()) + expect(report.exists()) + expect(not ("Pass severity gate failed:" not in result.stdout)) + expect(not ("NopInsertion=not-requested(expected <= clean)" not in result.stdout)) + + payload = json.loads(report.read_text(encoding="utf-8")) + expect(payload["passes"]["NopInsertion"]["symbolic_summary"]["severity"] == "not-requested") + expect( + payload["gate_evaluation"]["requested"]["require_pass_severity"] + == [{"pass_name": "NopInsertion", "max_severity": "clean"}] + ) + expect(not (payload["gate_evaluation"]["results"]["require_pass_severity_passed"] is not False)) + expect( + payload["gate_evaluation"]["results"]["require_pass_severity_failures"] + == ["NopInsertion=not-requested(expected <= clean)"] + ) + expect(not (payload["gate_evaluation"]["results"]["all_passed"] is not False)) + expect(payload["gate_failures"]["require_pass_severity_failure_count"] == 1) + expect( + payload["gate_failure_priority"] + == [ + { + "pass_name": "NopInsertion", + "failure_count": 1, + "strictest_expected_severity": "clean", + "failures": ["NopInsertion=not-requested(expected <= clean)"], + } + ] + ) + expect(payload["gate_failure_severity_priority"] == [{"severity": "clean", "failure_count": 1}]) + expect(payload["gate_failures"]["require_pass_severity_failures_by_expected_severity"] == {"clean": 1}) + expect(payload["summary"]["gate_failures"]["require_pass_severity_failure_count"] == 1) + expect(payload["summary"]["gate_failure_priority"] == payload["gate_failure_priority"]) + expect(payload["summary"]["gate_failure_severity_priority"] == payload["gate_failure_severity_priority"]) + + def test_cli_mutate_require_pass_severity_accepts_mutation_alias(self, ls_elf, tmp_path): + """Short mutation aliases should resolve to the concrete pass name.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "cli_mutate_pass_alias_ok.bin" + report = tmp_path / "cli_mutate_pass_alias_ok.report.json" + + result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "structural", + "--require-pass-severity", + "nop=not-requested", + "--seed", + "1337", + "-m", + "nop", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(result.returncode == 0) + expect(output_path.exists()) + expect(report.exists()) + expect(not ("Pass severity gate passed:" not in result.stdout)) + expect(not ("NopInsertion<=not-requested" not in result.stdout)) + + def test_cli_mutate_require_pass_severity_alias_can_fail(self, ls_elf, tmp_path): + """Short mutation aliases should produce the same failure semantics as pass names.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "cli_mutate_pass_alias_fail.bin" + report = tmp_path / "cli_mutate_pass_alias_fail.report.json" + + result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "structural", + "--require-pass-severity", + "nop=clean", + "--seed", + "1337", + "-m", + "nop", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(result.returncode == 1) + expect(output_path.exists()) + expect(report.exists()) + expect(not ("Pass severity gate failed:" not in result.stdout)) + expect(not ("NopInsertion=not-requested(expected <= clean)" not in result.stdout)) + + def test_cli_mutate_require_pass_severity_rejects_unknown_alias(self, ls_elf, tmp_path): + """Unknown aliases should fail fast with exit code 2 before mutating.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + output_path = tmp_path / "cli_mutate_pass_alias_invalid.bin" + report = tmp_path / "cli_mutate_pass_alias_invalid.report.json" + + result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(ls_elf), + "-o", + str(output_path), + "--report", + str(report), + "--validation-mode", + "structural", + "--require-pass-severity", + "definitely-missing=clean", + "--seed", + "1337", + "-m", + "nop", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(result.returncode == _EXPECTED_RESULT_RETURNCODE_2) + expect(not (output_path.exists())) + expect(not (report.exists())) + expect(not ("Invalid --require-pass-severity: definitely-missing=clean" not in result.stdout)) + + def test_cli_symbolic_blocks_limited_pass_without_override( + self, + deterministic_register_elf, + tmp_path, + ): + """Symbolic mode should fail fast for passes that declare limited symbolic support.""" + output_path = tmp_path / "register_blocked.bin" + + result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(deterministic_register_elf), + "-o", + str(output_path), + "--validation-mode", + "symbolic", + "--seed", + "1337", + "-m", + "register", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(result.returncode == _EXPECTED_RESULT_RETURNCODE_2_2) + expect(not ("symbolic validation is marked limited" not in result.stdout)) + expect(not ("--allow-limited-symbolic" not in result.stdout)) + expect(not (output_path.exists())) + + def test_cli_symbolic_allows_limited_pass_with_override( + self, + deterministic_register_elf, + tmp_path, + ): + """The explicit override should allow the limited symbolic pass to run.""" + output_path = tmp_path / "register_allowed.bin" + report_path = tmp_path / "register_allowed.report.json" + + result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(deterministic_register_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "symbolic", + "--allow-limited-symbolic", + "--seed", + "1337", + "-m", + "register", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(result.returncode == 0) + expect(not ("Limited symbolic coverage explicitly allowed" not in result.stdout)) + expect(output_path.exists()) + payload = json.loads(report_path.read_text(encoding="utf-8")) + expect( + not ( + payload["pass_support"]["RegisterSubstitution"]["validator_capabilities"]["symbolic"]["recommended"] + is not False + ) + ) + + def test_cli_symbolic_can_degrade_limited_pass_to_runtime( + self, + deterministic_register_elf, + tmp_path, + ): + """Limited symbolic support can degrade to runtime validation instead of blocking.""" + output_path = tmp_path / "register_runtime.bin" + report_path = tmp_path / "register_runtime.report.json" + + result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(deterministic_register_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "symbolic", + "--limited-symbolic-policy", + "degrade-runtime", + "--seed", + "1337", + "-m", + "register", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(result.returncode == 0) + expect(not ("Degrading validation mode from symbolic to runtime" not in result.stdout)) + expect(output_path.exists()) + payload = json.loads(report_path.read_text(encoding="utf-8")) + expect(payload["summary"]["requested_validation_mode"] == "symbolic") + expect(payload["summary"]["validation_mode"] == "runtime") + expect(payload["summary"]["degradation_roles"]["degradation-trigger"] == 1) + expect(payload["validation_policy"]["policy"] == "degrade-runtime") + expect(payload["validation_policy"]["reason"] == "limited-symbolic-support") + expect(payload["validation_policy"]["limited_passes"][0]["role"] == "degradation-trigger") + expect(not (payload["validation"]["runtime"]["passed"] not in {True, False})) + expect( + payload["passes"]["RegisterSubstitution"]["validation_context"]["requested_validation_mode"] == "symbolic" + ) + expect( + payload["passes"]["RegisterSubstitution"]["validation_context"]["effective_validation_mode"] == "runtime" + ) + expect(not (payload["passes"]["RegisterSubstitution"]["validation_context"]["degraded_execution"] is not True)) + expect( + not ( + payload["passes"]["RegisterSubstitution"]["validation_context"]["degradation_triggered_by_pass"] + is not True + ) + ) + expect(payload["passes"]["RegisterSubstitution"]["validation_context"]["role"] == "degradation-trigger") + + def test_cli_symbolic_can_degrade_limited_pass_to_structural( + self, + deterministic_register_elf, + tmp_path, + ): + """Limited symbolic support can degrade to structural validation instead of blocking.""" + output_path = tmp_path / "register_structural.bin" + report_path = tmp_path / "register_structural.report.json" + + result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "mutate", + str(deterministic_register_elf), + "-o", + str(output_path), + "--report", + str(report_path), + "--validation-mode", + "symbolic", + "--limited-symbolic-policy", + "degrade-structural", + "--seed", + "1337", + "-m", + "register", + ], + capture_output=True, + text=True, + timeout=90, + ) + + expect(result.returncode == 0) + expect(not ("Degrading validation mode from symbolic to structural" not in result.stdout)) + expect(output_path.exists()) + payload = json.loads(report_path.read_text(encoding="utf-8")) + expect(payload["summary"]["requested_validation_mode"] == "symbolic") + expect(payload["summary"]["validation_mode"] == "structural") + expect(payload["summary"]["degradation_roles"]["degradation-trigger"] == 1) + expect(payload["validation_policy"]["policy"] == "degrade-structural") + expect(payload["validation_policy"]["reason"] == "limited-symbolic-support") + expect(payload["validation_policy"]["limited_passes"][0]["role"] == "degradation-trigger") + expect("runtime" not in payload["validation"]) + expect(payload["passes"]["RegisterSubstitution"]["validation_context"]["role"] == "degradation-trigger") diff --git a/tests/integration/test_cli_report_views.py b/tests/integration/test_cli_report_views.py new file mode 100644 index 00000000..d1dfb5e4 --- /dev/null +++ b/tests/integration/test_cli_report_views.py @@ -0,0 +1,279 @@ +""" +Integration tests for CLI. +""" + +import importlib +import importlib.util +import json +import sys + +import pytest + +from r2morph import MorphEngine +from r2morph.core.engine_run import EngineRunOptions +from r2morph.mutations.register_substitution import RegisterSubstitutionPass +from tests.integration.cli_test_support import CLIIntegrationBase, _ReportFixturePass +from tests.utils.assertions import expect +from tests.utils.field_names import MUTATION_NAME_KEY +from tests.utils.process import run_command + +_EXPECTED_FAILURES_COUNT_NOPINSERTION_NOT_REQUESTED_EXP_2 = 2 +_EXPECTED_PRIORITY_0_FAILURE_COUNT_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2 = 2 +_EXPECTED_RESULT_RETURNCODE_2_2 = 2 + + +# Check if typer is available +try: + importlib_util = importlib.import_module("importlib.util") + + TYPER_AVAILABLE = importlib_util.find_spec("typer") is not None +except ImportError: + TYPER_AVAILABLE = False + + +@pytest.mark.skipif(not TYPER_AVAILABLE, reason="typer not installed") +class TestCLI(CLIIntegrationBase): + """CLI integration coverage for one cohesive command workflow.""" + + def test_cli_report_filters_on_engine_generated_report(self, ls_elf, tmp_path): + """Test report filters against a real engine-generated symbolic report.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + + report = tmp_path / "generated.report.json" + + with MorphEngine() as engine: + engine.load_binary(ls_elf).analyze() + engine.add_mutation(_ReportFixturePass()) + result = engine.run(EngineRunOptions(validation_mode="symbolic", report_path=report)) + + expect(report.exists()) + payload = json.loads(report.read_text(encoding="utf-8")) + expect(payload["mutations"]) + mutation = payload["mutations"][0] + symbolic_status = mutation["metadata"].get("symbolic_status") + expect(symbolic_status) + expect(not ("symbolic_issue_passes" not in payload["summary"])) + expect(not ("symbolic_coverage_by_pass" not in payload["summary"])) + expect(not ("symbolic_severity_by_pass" not in payload["summary"])) + expect(not ("symbolic_summary" not in payload["passes"]["ReportFixture"])) + expect(not ("severity" not in payload["passes"]["ReportFixture"]["symbolic_summary"])) + has_symbolic_issue = not mutation["metadata"].get("symbolic_observable_equivalent", False) and ( + mutation["metadata"].get("symbolic_observable_check_performed", False) + or symbolic_status + not in { + "real-binary-observables-match", + "shellcode-observables-match", + } + ) + if has_symbolic_issue: + expect(payload["summary"]["symbolic_issue_passes"]) + expect(payload["passes"]["ReportFixture"]["symbolic_summary"]["issues"]) + expect(payload["summary"]["symbolic_coverage_by_pass"]) + expect(not (payload["passes"]["ReportFixture"]["symbolic_summary"]["symbolic_requested"] < 1)) + expect(not (result["validation"]["symbolic"]["requested"] is not True)) + + summary_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--summary-only", + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + expect(summary_result.returncode == 0) + expect(not ("Symbolic Mutation Summary" not in summary_result.stdout)) + expect(not ("Severity Priority" not in summary_result.stdout)) + expect(not ("Pass Evidence" not in summary_result.stdout)) + if "RegisterSubstitution" in summary_result.stdout and "NopInsertion" in summary_result.stdout: + expect( + not (summary_result.stdout.index("RegisterSubstitution") >= summary_result.stdout.index("NopInsertion")) + ) + expect(not (has_symbolic_issue and "Passes With Symbolic Issues" not in summary_result.stdout)) + expect('"mutations"' not in summary_result.stdout) + + pass_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-pass", + "ReportFixture", + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + expect(pass_result.returncode == 0) + expect(not ("Pass Symbolic Summary" not in pass_result.stdout)) + expect(not ("Pass Evidence Summary" not in pass_result.stdout)) + expect(not ("severity=" not in pass_result.stdout)) + expect(not ('"pass_name": "ReportFixture"' not in pass_result.stdout)) + expect(not ('"report_filters": {' not in pass_result.stdout)) + expect(not ('"only_pass": "ReportFixture"' not in pass_result.stdout)) + + status_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-status", + symbolic_status, + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + expect(status_result.returncode == 0) + expect(not (f'"symbolic_status": "{symbolic_status}"' not in status_result.stdout)) + expect(not ('"report_filters": {' not in status_result.stdout)) + expect(not (f'"only_status": "{symbolic_status}"' not in status_result.stdout)) + + def test_cli_report_only_risky_passes_filters_real_risky_passes(self, deterministic_register_elf, tmp_path): + """Test `report --only-risky-passes` on a real report with symbolic mismatch evidence.""" + report = tmp_path / "register_risky.report.json" + filtered = tmp_path / "register_risky.filtered.json" + with MorphEngine(config={"seed": 1337}) as engine: + engine.load_binary(deterministic_register_elf).analyze() + engine.add_mutation( + RegisterSubstitutionPass( + config={ + "probability": 1.0, + "max_substitutions_per_function": 6, + "seed": 1337, + } + ) + ) + engine.run(EngineRunOptions(validation_mode="symbolic", seed=1337, report_path=report)) + + expect(report.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-risky-passes", + "--summary-only", + "--require-results", + "--output", + str(filtered), + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("Risky Pass Filter" not in report_result.stdout)) + expect(not ("RegisterSubstitution" not in report_result.stdout)) + + filtered_payload = json.loads(filtered.read_text(encoding="utf-8")) + expect(not (filtered_payload["report_filters"]["only_risky_passes"] is not True)) + expect(not ("RegisterSubstitution" not in filtered_payload["filtered_summary"]["risky_passes"])) + expect(not ("RegisterSubstitution" not in filtered_payload["filtered_summary"]["pass_risk_buckets"]["risky"])) + expect( + not ("RegisterSubstitution" not in filtered_payload["filtered_summary"]["pass_risk_buckets"]["symbolic"]) + ) + expect(filtered_payload["filtered_summary"]["pass_evidence"][0][MUTATION_NAME_KEY] == "RegisterSubstitution") + + def test_cli_report_only_symbolic_risk_filters_real_symbolic_risk(self, deterministic_register_elf, tmp_path): + """Test `report --only-symbolic-risk` on a real report with symbolic mismatch evidence.""" + report = tmp_path / "register_symbolic_risk.report.json" + filtered = tmp_path / "register_symbolic_risk.filtered.json" + + with MorphEngine(config={"seed": 1337}) as engine: + engine.load_binary(deterministic_register_elf).analyze() + engine.add_mutation( + RegisterSubstitutionPass( + config={ + "probability": 1.0, + "max_substitutions_per_function": 6, + "seed": 1337, + } + ) + ) + engine.run(EngineRunOptions(validation_mode="symbolic", seed=1337, report_path=report)) + + expect(report.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-symbolic-risk", + "--summary-only", + "--require-results", + "--output", + str(filtered), + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("Symbolic Risk Filter" not in report_result.stdout)) + expect(not ("RegisterSubstitution" not in report_result.stdout)) + + filtered_payload = json.loads(filtered.read_text(encoding="utf-8")) + expect(not (filtered_payload["report_filters"]["only_symbolic_risk"] is not True)) + expect(not ("RegisterSubstitution" not in filtered_payload["filtered_summary"]["symbolic_risk_passes"])) + expect(filtered_payload["filtered_summary"]["pass_evidence"][0][MUTATION_NAME_KEY] == "RegisterSubstitution") + + def test_cli_report_only_clean_passes_filters_real_clean_passes(self, ls_elf, tmp_path): + """Test `report --only-clean-passes` on a real report with clean symbolic evidence.""" + if not ls_elf.exists(): + pytest.skip("ELF binary not available") + report = tmp_path / "clean.report.json" + filtered = tmp_path / "clean.filtered.json" + + with MorphEngine() as engine: + engine.load_binary(ls_elf).analyze() + engine.add_mutation(_ReportFixturePass()) + engine.run(EngineRunOptions(validation_mode="off", report_path=report)) + + expect(report.exists()) + + report_result = run_command( + [ + sys.executable, + "-m", + "r2morph.cli", + "report", + "--only-clean-passes", + "--summary-only", + "--require-results", + "--output", + str(filtered), + str(report), + ], + capture_output=True, + text=True, + timeout=30, + ) + + expect(report_result.returncode == 0) + expect(not ("Clean Pass Filter" not in report_result.stdout)) + expect(not ("ReportFixture" not in report_result.stdout)) + + filtered_payload = json.loads(filtered.read_text(encoding="utf-8")) + expect(not (filtered_payload["report_filters"]["only_clean_passes"] is not True)) + expect(not ("ReportFixture" not in filtered_payload["filtered_summary"]["clean_passes"])) + expect(not ("ReportFixture" not in filtered_payload["filtered_summary"]["pass_risk_buckets"]["clean"])) + expect(filtered_payload["filtered_summary"]["pass_evidence"][0][MUTATION_NAME_KEY] == "ReportFixture") diff --git a/tests/integration/test_deterministic_elf_mutations.py b/tests/integration/test_deterministic_elf_mutations.py index 565e61ec..4851bb5e 100644 --- a/tests/integration/test_deterministic_elf_mutations.py +++ b/tests/integration/test_deterministic_elf_mutations.py @@ -3,6 +3,8 @@ """ import importlib.util +import json +from io import StringIO import pytest @@ -21,6 +23,7 @@ NopInsertionPass, RegisterSubstitutionPass, ) +from r2morph.utils.logging import setup_logging _EXPECTED_LEN_MUTATION_METADATA_GET_SYMBOLIC_BINARY_MUT_2 = 2 _EXPECTED_LEN_MUTATION_METADATA_GET_SYMBOLIC_BINARY_ORI_2 = 2 @@ -176,6 +179,23 @@ def test_deterministic_instruction_substitution_fixture_gets_symbolic_coverage( expect(not (report["summary"]["changed_bytes"] <= 0)) +def test_instruction_substitution_details_are_debug_only(deterministic_substitute_elf): + log_stream = StringIO() + setup_logging("DEBUG", stream=log_stream, environ={"R2MORPH_LOG_FORMAT": "json"}) + with MorphEngine(config={"seed": 1337}) as engine: + engine.load_binary(deterministic_substitute_elf).analyze() + engine.add_mutation( + InstructionSubstitutionPass( + config={"probability": 1.0, "max_substitutions_per_function": 8, "strict_size": True, "seed": 1337} + ) + ) + result = engine.run(EngineRunOptions(validation_mode="structural", seed=1337)) + + events = [json.loads(line) for line in log_stream.getvalue().splitlines()] + details = [event for event in events if event["message"].startswith("Substituted '")] + expect(result["total_mutations"] > 0 and details and all(event["level"] == "DEBUG" for event in details)) + + def test_deterministic_register_fixture_produces_real_substitutions(deterministic_register_elf): with MorphEngine(config={"seed": 1337}) as engine: engine.load_binary(deterministic_register_elf).analyze() diff --git a/tests/integration/test_elf_x86_32_native.py b/tests/integration/test_elf_x86_32_native.py index b6dd771f..bd34c31b 100644 --- a/tests/integration/test_elf_x86_32_native.py +++ b/tests/integration/test_elf_x86_32_native.py @@ -4,11 +4,15 @@ import platform import shutil +import struct from pathlib import Path import pytest +import unicorn +from unicorn import x86_const from r2morph.core.binary import Binary +from r2morph.mutations.block_reordering import BlockReorderingPass from r2morph.mutations.constant_unfolding import ConstantUnfoldingPass from r2morph.mutations.instruction_expansion import InstructionExpansionPass from r2morph.mutations.instruction_substitution import InstructionSubstitutionPass @@ -18,6 +22,18 @@ from tests.utils.assertions import expect from tests.utils.process import run_command +_ELF_PT_LOAD = 1 +_ELF_PAGE_SIZE = 0x1000 +_ELF32_ENTRY_OFFSET = 0x18 +_ELF32_PROGRAM_HEADER_OFFSET_OFFSET = 0x1C +_ELF32_PROGRAM_HEADER_SIZE_OFFSET = 0x2A +_ELF32_PROGRAM_HEADER_COUNT_OFFSET = 0x2C +_ELF32_STACK_BASE = 0x70000000 +_ELF32_STACK_SIZE = 0x10000 +_ELF32_SYSCALL_INTERRUPT = 0x80 +_ELF32_EXIT_SYSCALL = 1 +_ELF32_INSTRUCTION_LIMIT = 10_000 + def _build_x86_32_elf(tmp_path: Path, complex_fixture: bool = False) -> Path: compiler = shutil.which("clang") or shutil.which("gcc") @@ -66,16 +82,24 @@ def _build_x86_32_elf(tmp_path: Path, complex_fixture: bool = False) -> Path: "zero:\n" " addl %esi, %ecx\n" "done:\n" + " cmpl $42, %ecx\n" + " je validated\n" + " jmp finish\n" + "validated:\n" + " addl $0, %ecx\n" + "finish:\n" " movl %ecx, %eax\n" " ret\n" ".size _start, .-_start\n" ) source.write_text(complex_source if complex_fixture else basic_source, encoding="ascii") binary_path = tmp_path / "x86_32_exit" + linker_flags = ["-fuse-ld=lld"] if platform.system() == "Darwin" else [] run_command( [ compiler, *target_flags, + *linker_flags, "-nostdlib", "-static", "-Wl,-e,_start", @@ -91,6 +115,64 @@ def _build_x86_32_elf(tmp_path: Path, complex_fixture: bool = False) -> Path: return binary_path +def _emulate_elf32_exit_code(binary_path: Path) -> int: + raw = binary_path.read_bytes() + entry = struct.unpack_from(" None: + if interrupt != _ELF32_SYSCALL_INTERRUPT or emulator.reg_read(x86_const.UC_X86_REG_EAX) != _ELF32_EXIT_SYSCALL: + raise RuntimeError(f"unexpected ELF32 interrupt {interrupt}") + exit_codes.append(emulator.reg_read(x86_const.UC_X86_REG_EBX) & 0xFF) + emulator.emu_stop() + + machine.hook_add(unicorn.UC_HOOK_INTR, capture_exit) + machine.emu_start(entry, 0, count=_ELF32_INSTRUCTION_LIMIT) + if not exit_codes: + raise RuntimeError("ELF32 execution did not reach sys_exit") + return exit_codes[0] + + +def test_elf_x86_32_block_reordering_preserves_emulated_exit_code(tmp_path: Path) -> None: + if platform.system() == "Darwin" and shutil.which("ld.lld") is None: + pytest.skip("ld.lld is required to link an ELF32 fixture on macOS") + + binary_path = _build_x86_32_elf(tmp_path, complex_fixture=True) + original_exit_code = _emulate_elf32_exit_code(binary_path) + with Binary(binary_path, writable=True) as binary: + binary.analyze() + result = BlockReorderingPass({"probability": 1.0, "seed": 1, "max_functions": 10}).apply(binary) + + mutated_exit_code = _emulate_elf32_exit_code(binary_path) + expect(result["mutations_applied"] > 0, f"ELF x86-32 block reordering made no change: {result!r}") + expect( + (original_exit_code, mutated_exit_code) == (42, 42), + f"ELF x86-32 block reordering changed emulated execution: {original_exit_code} -> {mutated_exit_code}", + ) + + def _build_x86_32_expansion_elf(tmp_path: Path) -> Path: compiler = shutil.which("clang") or shutil.which("gcc") if compiler is None: diff --git a/tests/integration/test_mutations_x86_compiled_binary.py b/tests/integration/test_mutations_x86_compiled_binary.py index 007241c7..17ec6be1 100644 --- a/tests/integration/test_mutations_x86_compiled_binary.py +++ b/tests/integration/test_mutations_x86_compiled_binary.py @@ -67,6 +67,29 @@ def _build_x86_binary(tmp_dir: Path) -> Path: return output +def _build_block_reordering_x86_binary(tmp_dir: Path) -> Path: + source = tmp_dir / "x86_block_reordering.c" + source.write_text( + "#include \n" + "__attribute__((noinline)) int branchy(int value) {\n" + " int result;\n" + " if (value < 0) result = value * 3 + 7;\n" + " else result = value / 2 - 5;\n" + " if (result == 4) result += 11;\n" + " else result -= 3;\n" + " return result;\n" + "}\n" + "int main(int argc, char **argv) {\n" + " if (argc != 2) return 125;\n" + " return branchy((int)strtol(argv[1], 0, 10)) & 0xff;\n" + "}\n", + encoding="ascii", + ) + output = tmp_dir / "x86_block_reordering" + run_command(["/usr/bin/clang", "-arch", "x86_64", "-O0", "-fno-inline", "-o", str(output), str(source)]) + return output + + @pytest.fixture(scope="module") def x86_binary_path(tmp_path_factory: pytest.TempPathFactory) -> Path: if platform.system() != "Darwin": @@ -159,6 +182,34 @@ def test_x86_block_reordering_real(x86_binary_path: Path, tmp_path: Path): expect(not ("functions_processed" not in result)) +def test_x86_block_reordering_preserves_native_exit_code(tmp_path: Path): + if platform.system() != "Darwin": + pytest.skip("Mach-O x86_64 execution requires macOS") + if not _clang_available(): + pytest.skip("clang not available") + writable_path = _build_block_reordering_x86_binary(tmp_path) + inputs = (-9, -2, -1, 0, 2, 5, 10, 18, 21) + original = [run_command([writable_path, str(value)], text=True, timeout=30) for value in inputs] + expect( + len({result.returncode for result in original}) > 1, + "Mach-O control-flow inputs did not exercise distinct outcomes", + ) + + with Binary(writable_path, writable=True) as bin_obj: + bin_obj.analyze("aa") + result = BlockReorderingPass(config={"probability": 1.0, "max_functions": 5, "seed": 1}).apply(bin_obj) + + expect(result["mutations_applied"] > 0, f"Mach-O x86-64 block reordering made no change: {result!r}") + expect(CodeSigner().sign(writable_path, adhoc=True), "failed to re-sign mutated Mach-O") + mutated = [run_command([writable_path, str(value)], text=True, timeout=30) for value in inputs] + + expect( + [(result.returncode, result.stdout, result.stderr) for result in mutated] + == [(result.returncode, result.stdout, result.stderr) for result in original], + f"Mach-O x86-64 block reordering changed native execution: {original!r} -> {mutated!r}", + ) + + def test_x86_opaque_predicates_and_control_flow_detection(x86_binary_path: Path, tmp_path: Path): randomness.seed(3) writable_path = _copy_writable(tmp_path, x86_binary_path) diff --git a/tests/integration/test_nop_insertion_pe_real.py b/tests/integration/test_nop_insertion_pe_real.py index d706109c..d7efbea5 100644 --- a/tests/integration/test_nop_insertion_pe_real.py +++ b/tests/integration/test_nop_insertion_pe_real.py @@ -1,6 +1,8 @@ import platform import shutil +import subprocess from pathlib import Path +from typing import Any import pytest @@ -15,17 +17,26 @@ from tests.utils.assertions import expect from tests.utils.process import run_command -pytestmark = pytest.mark.skipif( - platform.system() == "Darwin" and platform.machine().lower() in {"arm64", "aarch64"}, - reason="the available PE runtime is x86_64-only on this ARM64 host", -) + +def _pe_toolchain() -> tuple[str, str | None]: + """Resolve a PE compiler and either native Windows or Wine execution.""" + is_windows = platform.system() == "Windows" + compiler = shutil.which("x86_64-w64-mingw32-gcc") + if compiler is None and is_windows: + compiler = shutil.which("gcc") + runtime = None if is_windows else shutil.which("wine") + if compiler is None or (not is_windows and runtime is None): + pytest.skip("a PE compiler and a native Windows or Wine runtime are required") + return compiler, runtime + + +def _run_pe(binary_path: Path, runtime: str | None) -> subprocess.CompletedProcess[Any]: + command = [str(binary_path)] if runtime is None else [runtime, str(binary_path)] + return run_command(command, timeout=30) def test_nop_insertion_pe_x86_64_preserves_repaired_integrity(tmp_path: Path) -> None: - compiler = shutil.which("x86_64-w64-mingw32-gcc") or shutil.which("gcc") - wine = None if platform.system() == "Windows" else shutil.which("wine") - if compiler is None or (platform.system() != "Windows" and wine is None): - pytest.skip("a PE compiler and a native or Wine runtime are required") + compiler, runtime = _pe_toolchain() source = tmp_path / "nop_sample.c" source.write_text( @@ -44,8 +55,7 @@ def test_nop_insertion_pe_x86_64_preserves_repaired_integrity(tmp_path: Path) -> ) binary_path = tmp_path / "nop_sample.exe" run_command([compiler, "-O0", "-fno-inline", "-o", str(binary_path), str(source)], check=True) - command = [str(binary_path)] if wine is None else [wine, str(binary_path)] - original_execution = run_command(command, timeout=30) + original_execution = _run_pe(binary_path, runtime) expect( original_execution.returncode == 0, f"generated PE fixture did not execute successfully: {original_execution.returncode}", @@ -70,7 +80,7 @@ def test_nop_insertion_pe_x86_64_preserves_repaired_integrity(tmp_path: Path) -> expect(handler.fix_checksum()) expect(handler.validate_integrity()[0]) - mutated_execution = run_command(command, timeout=30) + mutated_execution = _run_pe(binary_path, runtime) expect( mutated_execution.returncode == original_execution.returncode == 0, "PE NOP insertion changed the native execution result", @@ -78,10 +88,7 @@ def test_nop_insertion_pe_x86_64_preserves_repaired_integrity(tmp_path: Path) -> def test_instruction_substitution_pe_x86_64_preserves_native_execution(tmp_path: Path) -> None: - compiler = shutil.which("x86_64-w64-mingw32-gcc") - wine = shutil.which("wine") - if compiler is None or wine is None: - pytest.skip("PE compiler and Wine runtime are required") + compiler, runtime = _pe_toolchain() source = tmp_path / "substitution_sample.c" source.write_text( @@ -100,7 +107,7 @@ def test_instruction_substitution_pe_x86_64_preserves_native_execution(tmp_path: ) binary_path = tmp_path / "substitution_sample.exe" run_command([compiler, "-O0", "-fno-inline", "-o", str(binary_path), str(source)], check=True) - original_execution = run_command([wine, str(binary_path)], timeout=30) + original_execution = _run_pe(binary_path, runtime) expect(original_execution.returncode == 0, "generated PE fixture did not execute successfully") handler = PEHandler(binary_path) @@ -112,7 +119,7 @@ def test_instruction_substitution_pe_x86_64_preserves_native_execution(tmp_path: expect(handler.fix_checksum()) expect(handler.validate_integrity()[0]) - mutated_execution = run_command([wine, str(binary_path)], timeout=30) + mutated_execution = _run_pe(binary_path, runtime) expect( mutated_execution.returncode == original_execution.returncode == 0, "PE instruction substitution changed the native execution result", @@ -120,10 +127,7 @@ def test_instruction_substitution_pe_x86_64_preserves_native_execution(tmp_path: def test_register_substitution_pe_x86_64_preserves_native_execution(tmp_path: Path) -> None: - compiler = shutil.which("x86_64-w64-mingw32-gcc") or shutil.which("gcc") - wine = None if platform.system() == "Windows" else shutil.which("wine") - if compiler is None or (platform.system() != "Windows" and wine is None): - pytest.skip("a PE compiler and a native or Wine runtime are required") + compiler, runtime = _pe_toolchain() source = tmp_path / "register_sample.c" source.write_text( @@ -142,8 +146,7 @@ def test_register_substitution_pe_x86_64_preserves_native_execution(tmp_path: Pa ) binary_path = tmp_path / "register_sample.exe" run_command([compiler, "-O0", "-fno-inline", "-o", str(binary_path), str(source)], check=True) - command = [str(binary_path)] if wine is None else [wine, str(binary_path)] - original_execution = run_command(command, timeout=30) + original_execution = _run_pe(binary_path, runtime) expect(original_execution.returncode == 0, "generated PE fixture did not execute successfully") handler = PEHandler(binary_path) @@ -155,7 +158,7 @@ def test_register_substitution_pe_x86_64_preserves_native_execution(tmp_path: Pa expect(handler.fix_checksum()) expect(handler.validate_integrity()[0]) - mutated_execution = run_command(command, timeout=30) + mutated_execution = _run_pe(binary_path, runtime) expect( mutated_execution.returncode == original_execution.returncode == 0, "PE register substitution changed the native execution result", @@ -163,10 +166,7 @@ def test_register_substitution_pe_x86_64_preserves_native_execution(tmp_path: Pa def test_constant_unfolding_pe_x86_64_preserves_native_execution(tmp_path: Path) -> None: - compiler = shutil.which("x86_64-w64-mingw32-gcc") - wine = None if platform.system() == "Windows" else shutil.which("wine") - if compiler is None or (platform.system() != "Windows" and wine is None): - pytest.skip("a PE compiler and a native or Wine runtime are required") + compiler, runtime = _pe_toolchain() source = tmp_path / "constant_unfolding_sample.c" source.write_text( @@ -181,8 +181,7 @@ def test_constant_unfolding_pe_x86_64_preserves_native_execution(tmp_path: Path) ) binary_path = tmp_path / "constant_unfolding_sample.exe" run_command([compiler, "-O0", "-fno-inline", "-o", str(binary_path), str(source)], check=True) - command = [str(binary_path)] if wine is None else [wine, str(binary_path)] - original_execution = run_command(command, timeout=30) + original_execution = _run_pe(binary_path, runtime) expect(original_execution.returncode == 0, "generated PE fixture did not execute successfully") with Binary(binary_path, writable=True) as binary: @@ -194,7 +193,7 @@ def test_constant_unfolding_pe_x86_64_preserves_native_execution(tmp_path: Path) expect(handler.fix_checksum()) expect(handler.validate_integrity()[0]) - mutated_execution = run_command(command, timeout=30) + mutated_execution = _run_pe(binary_path, runtime) expect( mutated_execution.returncode == original_execution.returncode == 0, "PE constant unfolding changed the native execution result", @@ -202,10 +201,7 @@ def test_constant_unfolding_pe_x86_64_preserves_native_execution(tmp_path: Path) def test_tier1_pass_composition_pe_x86_64_preserves_native_execution(tmp_path: Path) -> None: - compiler = shutil.which("x86_64-w64-mingw32-gcc") or shutil.which("gcc") - wine = None if platform.system() == "Windows" else shutil.which("wine") - if compiler is None or (platform.system() != "Windows" and wine is None): - pytest.skip("a PE compiler and a native or Wine runtime are required") + compiler, runtime = _pe_toolchain() source = tmp_path / "composition_sample.c" source.write_text( @@ -221,8 +217,7 @@ def test_tier1_pass_composition_pe_x86_64_preserves_native_execution(tmp_path: P ) binary_path = tmp_path / "composition_sample.exe" run_command([compiler, "-O0", "-fno-inline", "-o", str(binary_path), str(source)], check=True) - command = [str(binary_path)] if wine is None else [wine, str(binary_path)] - original_execution = run_command(command, timeout=30) + original_execution = _run_pe(binary_path, runtime) expect(original_execution.returncode == 0, "generated PE fixture did not execute successfully") with Binary(binary_path, writable=True) as binary: @@ -236,7 +231,7 @@ def test_tier1_pass_composition_pe_x86_64_preserves_native_execution(tmp_path: P handler = PEHandler(binary_path) expect(handler.fix_checksum()) expect(handler.validate_integrity()[0]) - mutated_execution = run_command(command, timeout=30) + mutated_execution = _run_pe(binary_path, runtime) expect( first_result["mutations_applied"] > 0 and second_result["mutations_applied"] > 0 @@ -270,12 +265,7 @@ def test_instruction_substitution_pe_fixture_preserves_windows_exit_code(tmp_pat def test_instruction_expansion_pe_x86_64_preserves_native_execution(tmp_path: Path) -> None: - if platform.system() != "Windows": - pytest.skip("native PE fixture execution requires Windows") - - compiler = shutil.which("gcc") or shutil.which("x86_64-w64-mingw32-gcc") - if compiler is None: - pytest.skip("a PE compiler is required") + compiler, runtime = _pe_toolchain() source = tmp_path / "expansion_sample.c" source.write_text( "__attribute__((noinline)) int transform(int value) {\n" @@ -287,7 +277,7 @@ def test_instruction_expansion_pe_x86_64_preserves_native_execution(tmp_path: Pa ) binary_path = tmp_path / "expansion_sample.exe" run_command([compiler, "-O0", "-fno-inline", "-o", str(binary_path), str(source)], check=True) - original_execution = run_command([binary_path], timeout=30) + original_execution = _run_pe(binary_path, runtime) expect(original_execution.returncode == 0, "generated PE fixture did not execute successfully") with Binary(binary_path, writable=True) as binary: @@ -301,7 +291,7 @@ def test_instruction_expansion_pe_x86_64_preserves_native_execution(tmp_path: Pa expect(handler.fix_checksum()) expect(handler.validate_integrity()[0]) - mutated_execution = run_command([binary_path], timeout=30) + mutated_execution = _run_pe(binary_path, runtime) expect( mutated_execution.returncode == original_execution.returncode == 0, "PE instruction expansion changed the native execution result", @@ -309,12 +299,7 @@ def test_instruction_expansion_pe_x86_64_preserves_native_execution(tmp_path: Pa def test_string_obfuscation_pe_x86_64_preserves_native_execution(tmp_path: Path) -> None: - if platform.system() != "Windows": - pytest.skip("native PE fixture execution requires Windows") - - compiler = shutil.which("gcc") or shutil.which("x86_64-w64-mingw32-gcc") - if compiler is None: - pytest.skip("a PE compiler is required") + compiler, runtime = _pe_toolchain() source = tmp_path / "string_sample.c" source.write_text( '__attribute__((used)) static const char unreferenced_evidence_string[] = "platform-string-evidence";\n' @@ -324,7 +309,7 @@ def test_string_obfuscation_pe_x86_64_preserves_native_execution(tmp_path: Path) ) binary_path = tmp_path / "string_sample.exe" run_command([compiler, "-O0", "-fno-inline", "-o", str(binary_path), str(source)], check=True) - original_execution = run_command([binary_path], timeout=30) + original_execution = _run_pe(binary_path, runtime) expect(original_execution.returncode == 0, "generated PE fixture did not execute successfully") with Binary(binary_path, writable=True) as binary: @@ -337,7 +322,7 @@ def test_string_obfuscation_pe_x86_64_preserves_native_execution(tmp_path: Path) expect(result["mutations_applied"] > 0) expect(handler.fix_checksum()) expect(handler.validate_integrity()[0]) - mutated_execution = run_command([binary_path], timeout=30) + mutated_execution = _run_pe(binary_path, runtime) expect( mutated_execution.returncode == original_execution.returncode == 0, "PE string obfuscation changed the native execution result", diff --git a/tests/integration/test_pattern_substitution_pass.py b/tests/integration/test_pattern_substitution_pass.py index 30850adc..5aa40d67 100644 --- a/tests/integration/test_pattern_substitution_pass.py +++ b/tests/integration/test_pattern_substitution_pass.py @@ -7,7 +7,9 @@ """ import importlib.util +import json import shutil +from io import StringIO from pathlib import Path import pytest @@ -26,6 +28,7 @@ from r2morph.mutations.pattern_integration import PatternMatchIntegration from r2morph.mutations.pattern_substitution import PatternSubstitutionPass from r2morph.mutations.pattern_types import Instruction +from r2morph.utils.logging import setup_logging from tests.integration.elf_emulator import emulate_exit_code @@ -139,6 +142,21 @@ def test_pattern_substitution_records_each_instruction_once(deterministic_patter expect(len(start_addresses) == len(set(start_addresses))) +def test_pattern_substitution_instruction_details_are_debug_only(deterministic_pattern_subst_elf, tmp_path): + log_stream = StringIO() + setup_logging("DEBUG", stream=log_stream, environ={"R2MORPH_LOG_FORMAT": "json"}) + binary_path = tmp_path / "pattern_substitution" + shutil.copyfile(deterministic_pattern_subst_elf, binary_path) + + with Binary(binary_path, writable=True) as binary: + binary.analyze("aa") + stats = PatternSubstitutionPass({"probability": 1.0, "seed": 1337}).apply(binary) + + events = [json.loads(line) for line in log_stream.getvalue().splitlines()] + details = [event for event in events if event["message"].startswith("Pattern-substituted ")] + expect(stats["mutations_applied"] > 0 and details and all(event["level"] == "DEBUG" for event in details)) + + def test_pattern_substitution_preserves_flag_live_fixture_exit(tmp_path): source = Path("fixtures/dataset/elf_vm_arith_x86_64") mutated = tmp_path / "pattern_flag_live" diff --git a/tests/property/test_block_reordering_layout_properties.py b/tests/property/test_block_reordering_layout_properties.py new file mode 100644 index 00000000..e318f201 --- /dev/null +++ b/tests/property/test_block_reordering_layout_properties.py @@ -0,0 +1,38 @@ +"""Property checks for fallthrough-safe block ordering.""" + +from hypothesis import given +from hypothesis import strategies as st + +from r2morph.mutations.block_reordering_relocation import _BlockPlan, _draw_new_order +from tests._doubles.relocation_order_rng import ReverseShuffle +from tests.utils.assertions import expect + + +@given(chain_lengths=st.lists(st.integers(min_value=1, max_value=8), min_size=2, max_size=8)) +def test_block_reordering_order_preserves_fallthrough_chains(chain_lengths: list[int]) -> None: + blocks: list[dict[str, int]] = [] + plans: dict[int, _BlockPlan] = {} + next_address = 0x1000 + for chain_length in chain_lengths: + chain = [next_address + index * 4 for index in range(chain_length)] + blocks.extend({"addr": address} for address in chain) + for index, address in enumerate(chain): + successor = chain[index + 1] if index + 1 < chain_length else None + plans[address] = _BlockPlan(address, [], successor) + next_address += chain_length * 4 + + reordered = _draw_new_order(blocks, plans, ReverseShuffle()) + result = blocks if reordered is None else reordered + result_addresses = [block["addr"] for block in result] + result_positions = {address: index for index, address in enumerate(result_addresses)} + original_addresses = [block["addr"] for block in blocks] + + expect( + result_addresses[: chain_lengths[0]] == original_addresses[: chain_lengths[0]] + and sorted(result_addresses) == original_addresses + and all( + result_positions[plan.fallthrough] == result_positions[address] + 1 + for address, plan in plans.items() + if plan.fallthrough is not None + ) + ) diff --git a/tests/unit/test_arch_support.py b/tests/unit/test_arch_support.py index ba25c79e..c9253d60 100644 --- a/tests/unit/test_arch_support.py +++ b/tests/unit/test_arch_support.py @@ -52,12 +52,17 @@ def test_classify_x86_32_target(self): expect(result["tier"] == "prolonged-experimental") expect(result["architecture"] == "x86") - def test_classify_arm32_target(self): - """Test ARM32 target classification.""" + def test_classify_arm32_target_marks_it_experimental(self): + """ARM32 remains experimental instead of appearing unsupported.""" result = classify_target_support("ELF", "arm", 32) - expect(result["tier"] == "unsupported") - expect(not ("architecture" not in result)) + expect(result["tier"] == "prolonged-experimental") + + def test_classify_arm32_target_uses_canonical_architecture(self): + """ARM32 reports the canonical identifier used by the support matrix.""" + result = classify_target_support("ELF", "arm", 32) + + expect(result["architecture"] == "arm32") def test_classify_arm64_target(self): """Test ARM64 target classification.""" diff --git a/tests/unit/test_binary_rewriter_helpers_more.py b/tests/unit/test_binary_rewriter_helpers_more.py index 0d45296f..e878648a 100644 --- a/tests/unit/test_binary_rewriter_helpers_more.py +++ b/tests/unit/test_binary_rewriter_helpers_more.py @@ -1,9 +1,12 @@ +import json +from io import StringIO from pathlib import Path import pytest from r2morph.core.binary import Binary from r2morph.devirtualization.binary_rewriter import BinaryFormat, BinaryRewriter, CodePatch, RewriteOperation +from r2morph.utils.logging import setup_logging from tests.utils.assertions import expect _EXPECTED_LEN_ASSEMBLED_2 = 2 @@ -68,3 +71,18 @@ def test_binary_rewriter_helpers_with_real_binary(tmp_path: Path): checks = rewriter._perform_integrity_checks(str(output_path)) expect(not (checks["file_exists"] is not True)) expect(not (checks["valid_pe_header"] is not True)) + + +def test_binary_rewriter_assembly_error_keeps_sample_in_debug_only(): + log_stream = StringIO() + setup_logging("DEBUG", stream=log_stream, environ={"R2MORPH_LOG_FORMAT": "json"}) + rewriter = BinaryRewriter() + rewriter.arch = "amd64" + expect(rewriter._initialize_codegen()) + expect(rewriter._assemble_instructions(["invalid_secret_marker"]) == b"") + + events = [json.loads(line) for line in log_stream.getvalue().splitlines()] + error = next(event for event in events if event.get("event_name") == "devirtualization.assembly.failed") + detail = next(event for event in events if "invalid_secret_marker" in event["message"]) + expect(error["level"] == "ERROR" and "invalid_secret_marker" not in error["message"]) + expect(detail["level"] == "DEBUG") diff --git a/tests/unit/test_block_reordering_relocation_order.py b/tests/unit/test_block_reordering_relocation_order.py new file mode 100644 index 00000000..f7e66d2f --- /dev/null +++ b/tests/unit/test_block_reordering_relocation_order.py @@ -0,0 +1,39 @@ +from r2morph.mutations.block_reordering_relocation import _BlockPlan, _draw_new_order +from tests._doubles.relocation_order_rng import ReverseShuffle +from tests.utils.assertions import expect + + +def _blocks() -> list[dict[str, int]]: + return [{"addr": address} for address in (0x1000, 0x1010, 0x1020, 0x1030, 0x1040)] + + +def _plans(blocks: list[dict[str, int]]) -> dict[int, _BlockPlan]: + addresses = [block["addr"] for block in blocks] + return { + address: _BlockPlan(address, [], addresses[index + 1] if index == 0 else None) + for index, address in enumerate(addresses) + } + + +def test_draw_new_order_keeps_fallthrough_chain_adjacent() -> None: + blocks = _blocks() + + reordered = _draw_new_order(blocks, _plans(blocks), ReverseShuffle()) + + expect(reordered is not None and [block["addr"] for block in reordered[:2]] == [0x1000, 0x1010]) + + +def test_draw_new_order_shuffles_independent_groups() -> None: + blocks = _blocks() + + reordered = _draw_new_order(blocks, _plans(blocks), ReverseShuffle()) + + expect(reordered is not None and reordered[2:] != blocks[2:]) + + +def test_draw_new_order_retains_every_block() -> None: + blocks = _blocks() + + reordered = _draw_new_order(blocks, _plans(blocks), ReverseShuffle()) + + expect(reordered is not None and {block["addr"] for block in reordered} == {block["addr"] for block in blocks}) diff --git a/tests/unit/test_ci_toolchain_contract.py b/tests/unit/test_ci_toolchain_contract.py new file mode 100644 index 00000000..fba626e2 --- /dev/null +++ b/tests/unit/test_ci_toolchain_contract.py @@ -0,0 +1,34 @@ +"""Contracts for tools required by the CI corpus tests.""" + +import tomllib +from pathlib import Path + +from tests.utils.assertions import expect + +_ROOT = Path(__file__).resolve().parents[2] + + +def test_ubuntu_ci_installs_clang_for_compiler_generated_corpus() -> None: + workflow = (_ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8") + ubuntu_setup = workflow.split("Install system dependencies (Ubuntu)", 1)[1].split( + "Install system dependencies (macOS)", 1 + )[0] + + expect("build-essential clang gcc-arm-linux-gnueabihf" in ubuntu_setup) + + +def test_unicorn_is_unconditionally_installed_by_test_extras() -> None: + requirements = (_ROOT / "requirements.txt").read_text(encoding="utf-8").splitlines() + project = tomllib.loads((_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + optional_dependencies = project["project"]["optional-dependencies"] + expected = ["unicorn>=2.0.0"] + + unicorn_requirements = [line for line in requirements if line.lower().startswith("unicorn")] + unicorn_extras = [ + dependency + for extra in ("dev", "devirtualization", "all") + for dependency in optional_dependencies[extra] + if dependency.lower().startswith("unicorn") + ] + + expect(unicorn_requirements == expected and unicorn_extras == expected * 3) diff --git a/tests/unit/test_critical_skip_gate.py b/tests/unit/test_critical_skip_gate.py new file mode 100644 index 00000000..c59644d2 --- /dev/null +++ b/tests/unit/test_critical_skip_gate.py @@ -0,0 +1,41 @@ +"""Regression checks for the stable product smoke skip gate.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml + +from scripts.check_critical_skips import skipped_test_count +from tests.utils.assertions import expect + +_ROOT = Path(__file__).resolve().parents[2] + + +def test_critical_skip_gate_accepts_junit_report_without_skips(tmp_path: Path) -> None: + report = tmp_path / "junit.xml" + report.write_text('', encoding="utf-8") + + expect(skipped_test_count(report) == 0) + + +def test_critical_skip_gate_rejects_junit_report_with_skipped_test(tmp_path: Path) -> None: + report = tmp_path / "junit.xml" + report.write_text( + '', + encoding="utf-8", + ) + + expect(skipped_test_count(report) == 1) + + +def test_ci_product_smoke_runs_critical_skip_gate() -> None: + workflow: dict[str, Any] = yaml.safe_load((_ROOT / ".github" / "workflows" / "ci.yml").read_text()) + steps = workflow["jobs"]["product-smoke-tests"]["steps"] + smoke_step = next(step for step in steps if step.get("name") == "Run product smoke tests") + + expect( + '--junitxml="$RUNNER_TEMP/stable-product-smoke.xml"' in smoke_step["run"] + and 'scripts/check_critical_skips.py "$RUNNER_TEMP/stable-product-smoke.xml"' in smoke_step["run"] + ) diff --git a/tests/unit/test_dependency_installation_contract.py b/tests/unit/test_dependency_installation_contract.py index d404181d..7e058791 100644 --- a/tests/unit/test_dependency_installation_contract.py +++ b/tests/unit/test_dependency_installation_contract.py @@ -8,8 +8,6 @@ from tests.utils.assertions import expect _ROOT = Path(__file__).resolve().parents[2] -_UNSUPPORTED_UNICORN_ON_MAC_ARM = 'unicorn>=2.0.0; platform_system != "Darwin" or platform_machine != "arm64"' -_EXPECTED_UNICORN_MANIFEST_COUNT = 3 _ANGR_BITSTRING_CONTRACT = 'bitstring>=4.4.0,<5; platform_system != "Windows"' _ANGR_PYCPARSER_CONTRACT = 'pycparser>=3.1,<3.2; platform_system != "Windows"' _EXPECTED_ANGR_COMPATIBILITY_MANIFEST_COUNT = 2 @@ -29,18 +27,6 @@ def test_supported_dependency_manifests_include_required_analyzer_packages() -> ) -def test_supported_dependency_manifests_skip_unicorn_on_macos_arm64() -> None: - requirements = (_ROOT / "requirements.txt").read_text(encoding="utf-8") - project = tomllib.loads((_ROOT / "pyproject.toml").read_text(encoding="utf-8")) - optional_dependencies = project["project"]["optional-dependencies"] - optional_text = "\n".join(dependency for group in optional_dependencies.values() for dependency in group) - - expect( - requirements.count(_UNSUPPORTED_UNICORN_ON_MAC_ARM) == 1 - and optional_text.count(_UNSUPPORTED_UNICORN_ON_MAC_ARM) == _EXPECTED_UNICORN_MANIFEST_COUNT - ) - - def test_supported_dependency_manifests_keep_angr_bitstring_api_compatible() -> None: requirements = (_ROOT / "requirements.txt").read_text(encoding="utf-8") project = tomllib.loads((_ROOT / "pyproject.toml").read_text(encoding="utf-8")) diff --git a/tests/unit/test_enhanced_analysis_orchestrator.py b/tests/unit/test_enhanced_analysis_orchestrator.py index 35327168..0100fe55 100644 --- a/tests/unit/test_enhanced_analysis_orchestrator.py +++ b/tests/unit/test_enhanced_analysis_orchestrator.py @@ -1,8 +1,11 @@ +import json +from io import StringIO from pathlib import Path import pytest from r2morph.analysis.enhanced_analyzer import AnalysisOptions, EnhancedAnalysisOrchestrator +from r2morph.utils.logging import setup_logging, shutdown_logging from tests.utils.assertions import expect pytestmark = [pytest.mark.experimental] @@ -55,3 +58,26 @@ def test_enhanced_analysis_orchestrator_paths(tmp_path): orchestrator._cleanup() if bin_obj is not None: bin_obj.__exit__(None, None, None) + + +def test_detect_only_analysis_logs_correlated_completion() -> None: + stream = StringIO() + setup_logging(stream=stream, environ={"R2MORPH_LOG_FORMAT": "json", "R2MORPH_LOG_LEVEL": "DEBUG"}) + orchestrator = EnhancedAnalysisOrchestrator(Path("fixtures/dataset/elf_x86_64")) + try: + orchestrator.analyze(AnalysisOptions(detect_only=True)) + finally: + shutdown_logging() + + all_events = [json.loads(line) for line in stream.getvalue().splitlines()] + operation_events = [ + event + for event in all_events + if event.get("event_name") in {"analysis.enhanced.started", "analysis.enhanced.completed"} + ] + expect( + [event["event_name"] for event in operation_events] + == ["analysis.enhanced.started", "analysis.enhanced.completed"] + and operation_events[0]["correlation_id"] == operation_events[1]["correlation_id"] + and operation_events[1]["duration_ms"] >= 0 + ) diff --git a/tests/unit/test_enhanced_analyzer_phases_contract.py b/tests/unit/test_enhanced_analyzer_phases_contract.py index 1326b61e..d5d46408 100644 --- a/tests/unit/test_enhanced_analyzer_phases_contract.py +++ b/tests/unit/test_enhanced_analyzer_phases_contract.py @@ -1,3 +1,5 @@ +import json +from io import StringIO from pathlib import Path from rich.console import Console @@ -11,6 +13,7 @@ run_iterative_simplification, run_symbolic_analysis, ) +from r2morph.utils.logging import setup_logging, shutdown_logging from tests.utils.assertions import expect @@ -30,3 +33,22 @@ def test_enhanced_analyzer_phase_helpers_basic_flow(tmp_path): expect(rewrite_result is None or isinstance(rewrite_result, str)) finally: binary.__exit__(None, None, None) + + +def test_cfo_simplification_failure_emits_structured_warning() -> None: + stream = StringIO() + setup_logging(stream=stream, environ={"R2MORPH_LOG_FORMAT": "json"}) + try: + result = run_cfo_simplification(None, Console(record=True), AnalysisResults()) + finally: + shutdown_logging() + + event = json.loads(stream.getvalue().splitlines()[0]) + expect( + result == 0 + and event["event_name"] == "analysis.cfo_simplification.failed" + and event["operation"] == "analysis.cfo_simplification" + and event["outcome"] == "degraded" + and event["error_type"] == "AttributeError" + and "stack_trace" in event + ) diff --git a/tests/unit/test_import_isolation.py b/tests/unit/test_import_isolation.py index 2a7be19f..af998807 100644 --- a/tests/unit/test_import_isolation.py +++ b/tests/unit/test_import_isolation.py @@ -115,6 +115,7 @@ def test_optional_angr_import_boundary_handles_known_external_warning() -> None: "from r2morph.adapters.angr_import import import_angr_modules\n" "stack = import_angr_modules(claripy=True, project=True, cfg_fast=True, exploration_technique=True)\n" "assert stack.available, 'angr stack should import when installed'\n" + "assert stack.claripy is __import__('claripy'), 'symbolic expressions must use the matching claripy module'\n" ) result = run_command( [sys.executable, "-W", "error", "-c", probe], diff --git a/tests/unit/test_independent_analysis_evidence_contract.py b/tests/unit/test_independent_analysis_evidence_contract.py new file mode 100644 index 00000000..f03da353 --- /dev/null +++ b/tests/unit/test_independent_analysis_evidence_contract.py @@ -0,0 +1,45 @@ +"""Regression checks for bounded independent-analysis evidence artifacts.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from tests.utils.assertions import expect + +_ROOT = Path(__file__).resolve().parents[2] +_REPORT_PATH = _ROOT / "docs" / "protection-independent-analysis-2026-11-10.json" +_EXPECTED_PROTECTED_FUNCTION_COUNT = 2 + + +def _report() -> dict[str, Any]: + return json.loads(_REPORT_PATH.read_text(encoding="utf-8")) + + +def test_independent_analysis_report_binds_to_the_checked_in_fixture() -> None: + report = _report() + fixture = _ROOT / report["fixture"]["path"] + fixture_sha256 = hashlib.sha256(fixture.read_bytes()).hexdigest() + + expect(fixture_sha256 == report["fixture"]["sha256"] == report["original"]["sha256"]) + + +def test_independent_analysis_report_keeps_the_observation_fixture_bounded() -> None: + report = _report() + + expect( + report["measurement"] == "targeted-independent-static-analysis-pair" + and report["original"]["function_count"] == 1 + and report["protected"]["function_count"] == _EXPECTED_PROTECTED_FUNCTION_COUNT + ) + + +def test_independent_analysis_report_does_not_claim_runtime_or_general_resistance() -> None: + report = _report() + + expect( + report["runtime_equivalence"] == "not measured" + and any("does not establish general resistance" in item for item in report["limitations"]) + ) diff --git a/tests/unit/test_independent_review_contract.py b/tests/unit/test_independent_review_contract.py index a24a6570..14b98b0d 100644 --- a/tests/unit/test_independent_review_contract.py +++ b/tests/unit/test_independent_review_contract.py @@ -4,6 +4,7 @@ from scripts.independent_review import ( _EXPECTED_BENCHMARK_TOOLS, + _human_review_signoff, _review_live_adversarial_benchmark, review, ) @@ -67,11 +68,13 @@ def test_independent_review_rejects_live_campaign_without_ghidra(tmp_path: Path) def test_independent_review_passes_published_artifacts() -> None: report = review(_ROOT) + signoff = json.loads((_ROOT / "docs" / "human-review-signoff.json").read_text(encoding="utf-8")) expect( report["passed"] is True - and report["human_signoff"] == "not-attested" - and report["release_decision"]["status"] == "block-vm-milestone" + and report["human_signoff"] == "attested" + and report["human_review"] == signoff + and report["release_decision"]["status"] == "signoff-recorded" ) @@ -115,3 +118,44 @@ def test_independent_review_validates_fppackedidxnb_regression_artifact() -> Non checks = {check["name"]: check["status"] for check in report["checks"]} expect(checks["fppackedidxnb_regression_evidence"] == "passed") + + +def test_review_packet_records_independent_human_approval() -> None: + packet = (_ROOT / "docs" / "independent-review-packet.md").read_text(encoding="utf-8") + normalized_packet = " ".join(packet.split()) + + expect( + "Result: APPROVE" in normalized_packet + and "Findings: None reported" in normalized_packet + and "reviewer acted independently of the authoring team" in normalized_packet + and "specialist experience in binary rewriting and virtualization" in normalized_packet + and "Reviewed commit: `a111f923df43a66826b71ab70db8a484532fbf3b`" in normalized_packet + ) + + +def test_independent_review_rejects_malformed_signoff_record(tmp_path: Path) -> None: + signoff_path = tmp_path / "docs" / "human-review-signoff.json" + signoff_path.parent.mkdir() + signoff_path.write_text( + json.dumps( + { + "schema_version": 1, + "reviewer": "Reviewer", + "review_date_utc": "2026-11-08", + "result": "APPROVE", + "reviewed_commit": "not-a-commit", + "independent": True, + "independence_confirmation": "confirmed", + "findings": [], + } + ), + encoding="utf-8", + ) + + rejected = False + try: + _human_review_signoff(tmp_path) + except ValueError: + rejected = True + + expect(rejected) diff --git a/tests/unit/test_observability_logging.py b/tests/unit/test_observability_logging.py index 2003b497..e7ef93b9 100644 --- a/tests/unit/test_observability_logging.py +++ b/tests/unit/test_observability_logging.py @@ -7,6 +7,7 @@ import logging import os import sys +from collections.abc import Iterator, Mapping from concurrent.futures import ThreadPoolExecutor from io import StringIO from pathlib import Path @@ -16,8 +17,10 @@ from r2morph.adapters.process import run_process from r2morph.utils.logging import ( + _MAX_COLLECTION_ITEMS, LoggingConfig, ProcessOwnerFilter, + _safe_value, logging_context, observed_operation, setup_logging, @@ -34,6 +37,27 @@ _MAX_SAFE_MESSAGE_LENGTH_4200 = 4200 +class _IterationBoundedMapping(Mapping[str, str]): + def __getitem__(self, key: str) -> str: + return key + + def __iter__(self) -> Iterator[str]: + yield from (str(index) for index in range(_MAX_COLLECTION_ITEMS + 1)) + raise AssertionError("mapping iteration exceeded the logging collection cap") + + def __len__(self) -> int: + return _MAX_COLLECTION_ITEMS + 1 + + +class _IterationBoundedList(list[str]): + def __iter__(self) -> Iterator[str]: + yield from (str(index) for index in range(_MAX_COLLECTION_ITEMS + 1)) + raise AssertionError("list iteration exceeded the logging collection cap") + + def __len__(self) -> int: + return _MAX_COLLECTION_ITEMS + 1 + + @pytest.fixture(autouse=True) def close_logging_handlers_after_test() -> Any: yield @@ -234,6 +258,19 @@ def test_formatter_redacts_secrets_paths_and_control_characters() -> None: ) +def test_formatter_redacts_compound_secret_assignments() -> None: + stream = StringIO() + logger = setup_logging(stream=stream, environ={"R2MORPH_LOG_FORMAT": "json"}) + + logger.error("client_secret=alpha secret_key=beta private_key=gamma oauth_token=delta id_token=epsilon") + + message = _json_events(stream)[0]["message"] + expect( + message == "client_secret= secret_key= private_key= " + "oauth_token= id_token=" + ) + + def test_formatter_bounds_untrusted_message_size() -> None: stream = StringIO() logger = setup_logging(stream=stream, environ={"R2MORPH_LOG_FORMAT": "json"}) @@ -244,6 +281,63 @@ def test_formatter_bounds_untrusted_message_size() -> None: expect(len(message) < _MAX_SAFE_MESSAGE_LENGTH_4200 and message.endswith("")) +def test_safe_value_bounds_mapping_iteration() -> None: + safe_mapping = _safe_value(_IterationBoundedMapping()) + + expect( + safe_mapping + == { + **{str(index): str(index) for index in range(_MAX_COLLECTION_ITEMS)}, + "_truncated_items": 1, + } + ) + + +def test_safe_value_redacts_secret_assignments_in_mapping_keys() -> None: + safe_mapping = _safe_value({"password=hunter2": "private"}) + + expect(json.dumps(safe_mapping) == '{"password=": ""}') + + +def test_safe_value_redacts_compound_secret_field_names() -> None: + safe_mapping = _safe_value( + { + "client_secret": "alpha", + "secret_key": "beta", + "oauth-token": "gamma", + "id_token": "delta", + } + ) + + expect( + safe_mapping + == { + "client_secret": "", + "secret_key": "", + "oauth-token": "", + "id_token": "", + } + ) + + +def test_safe_value_sanitizes_path_basename() -> None: + safe_path = _safe_value(Path("token=pathsecret\nforged")) + + expect(safe_path == "\\nforged>") + + +def test_safe_value_bounds_sequence_iteration() -> None: + safe_values = _safe_value(_IterationBoundedList()) + + expect( + safe_values + == [ + *[str(index) for index in range(_MAX_COLLECTION_ITEMS)], + "", + ] + ) + + def _write_concurrent_events(logger: logging.Logger, worker_id: int) -> None: with logging_context(correlation_id=f"worker-{worker_id}", operation="parallel.run"): for sequence in range(10): diff --git a/tests/unit/test_one_zero_release_gate.py b/tests/unit/test_one_zero_release_gate.py new file mode 100644 index 00000000..afd5055e --- /dev/null +++ b/tests/unit/test_one_zero_release_gate.py @@ -0,0 +1,88 @@ +"""Regression tests for the stable-major evidence gate.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from scripts.check_release_contract import _check_one_zero_release_gate +from tests.utils.assertions import expect + +_ROOT = Path(__file__).resolve().parents[2] +_READINESS = _ROOT / "docs" / "one-zero-readiness.json" +_PASS_MATURITY = _ROOT / "docs" / "pass-maturity.md" +_RELEASE_BLOCKERS = _ROOT / "docs" / "release-blockers.md" +_CRITERIA = ( + "all_named_adversarial_analyzers", + "complete_compatibility_contract", + "cross_platform_parity", + "documented_resistance_scope", + "published_cost_results", + "published_correctness_results", + "published_resistance_results", + "stable_api_cli", +) + + +def test_one_zero_gate_allows_alpha_while_readiness_is_open() -> None: + expect(_check_one_zero_release_gate("0.4.0-alpha.3", _READINESS, _ROOT) is None) + + +def test_one_zero_gate_blocks_stable_major_with_open_readiness() -> None: + rejected = False + try: + _check_one_zero_release_gate("1.0.0", _READINESS, _ROOT) + except ValueError as error: + rejected = "blocked by incomplete 1.0 criteria" in str(error) + + expect(rejected) + + +def test_readiness_keeps_general_resistance_results_open_when_scope_is_documented() -> None: + readiness = json.loads(_READINESS.read_text(encoding="utf-8")) + pass_maturity = " ".join(_PASS_MATURITY.read_text(encoding="utf-8").split()) + release_blockers = " ".join(_RELEASE_BLOCKERS.read_text(encoding="utf-8").split()) + criteria = readiness["criteria"] + + expect( + criteria["documented_resistance_scope"]["status"] == "complete" + and criteria["published_resistance_results"]["status"] == "open" + and "published_resistance_results" in readiness["blocking_criteria"] + and "does not establish general resistance to professional reverse engineering" in pass_maturity + and "close the 1.0 `published_resistance_results` criterion" in pass_maturity + and "does not establish general resistance to professional reverse engineering" in release_blockers + ) + + +def test_one_zero_gate_accepts_complete_linked_evidence(tmp_path: Path) -> None: + criteria: dict[str, Any] = {} + for name in _CRITERIA: + evidence_path = tmp_path / f"{name}.md" + evidence_path.write_text("verified evidence\n", encoding="utf-8") + criteria[name] = {"status": "complete", "evidence": [evidence_path.name]} + report = { + "schema_version": 1, + "status": "ready", + "blocking_criteria": [], + "criteria": criteria, + } + report_path = tmp_path / "readiness.json" + report_path.write_text(json.dumps(report), encoding="utf-8") + + expect(_check_one_zero_release_gate("1.0.0", report_path, tmp_path) is None) + + +def test_one_zero_gate_rejects_evidence_paths_outside_the_readiness_root(tmp_path: Path) -> None: + report = json.loads(_READINESS.read_text(encoding="utf-8")) + report["criteria"][_CRITERIA[0]]["evidence"] = ["../outside.md"] + report_path = tmp_path / "readiness.json" + report_path.write_text(json.dumps(report), encoding="utf-8") + + rejected = False + try: + _check_one_zero_release_gate("0.4.0-alpha.3", report_path, tmp_path) + except ValueError as error: + rejected = "missing or unsafe" in str(error) + + expect(rejected) diff --git a/tests/unit/test_platform_evidence_contract.py b/tests/unit/test_platform_evidence_contract.py index c161470b..496e590e 100644 --- a/tests/unit/test_platform_evidence_contract.py +++ b/tests/unit/test_platform_evidence_contract.py @@ -81,6 +81,10 @@ def test_platform_evidence_uses_native_composition_contracts() -> None: ) +def test_macos_arm64_evidence_excludes_x86_string_obfuscation() -> None: + expect("StringObfuscation" not in _PASS_CASES["macos-arm64"]) + + def test_platform_evidence_rejects_report_missing_required_case(tmp_path: Path) -> None: for platform_name in _REQUIRED_CASES: (tmp_path / platform_name).mkdir() diff --git a/tests/unit/test_process_adapter.py b/tests/unit/test_process_adapter.py index 18360fe7..805b40c9 100644 --- a/tests/unit/test_process_adapter.py +++ b/tests/unit/test_process_adapter.py @@ -1,6 +1,5 @@ """Real-process contract for the external process adapter.""" -import os import sys import time from pathlib import Path @@ -34,7 +33,7 @@ def test_run_process_executes_shebang_script_without_shell(tmp_path: Path) -> No expect(result.stdout_text == "ok") -def test_run_process_timeout_terminates_child_process_group(tmp_path: Path) -> None: +def test_run_process_timeout_terminates_descendant_processes(tmp_path: Path) -> None: marker = tmp_path / "child-survived" child_code = "import pathlib, time; " f"time.sleep(0.5); pathlib.Path({str(marker)!r}).write_text('survived')" parent_code = ( @@ -45,4 +44,4 @@ def test_run_process_timeout_terminates_child_process_group(tmp_path: Path) -> N run_process([Path(sys.executable), "-c", parent_code], timeout=0.1) time.sleep(0.7) - expect(marker.exists() is (os.name != "posix")) + expect(not marker.exists()) diff --git a/tests/unit/test_public_surface_contract.py b/tests/unit/test_public_surface_contract.py new file mode 100644 index 00000000..88334410 --- /dev/null +++ b/tests/unit/test_public_surface_contract.py @@ -0,0 +1,82 @@ +"""Regression contracts for the supported package and command-line surfaces.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +from typer.main import get_command + +import r2morph +from r2morph.cli import app +from tests.utils.assertions import expect + +_ROOT = Path(__file__).resolve().parents[2] +_PUBLIC_EXPORTS = ("Binary", "MorphEngine", "Pipeline", "__version__") +_PUBLIC_COMMANDS = { + "analyze", + "cache", + "diff", + "experimental", + "functions", + "morph", + "mutate", + "report", + "validate", + "version", +} +_MUTATE_LONG_OPTIONS = { + "--aggressive", + "--allow-limited-symbolic", + "--cache", + "--clear-cache", + "--force", + "--format", + "--limited-symbolic-policy", + "--min-severity", + "--mutation", + "--output", + "--report", + "--require-pass-severity", + "--rollback-policy", + "--runtime-compare-files", + "--runtime-corpus", + "--runtime-normalize-whitespace", + "--runtime-timeout", + "--seed", + "--validation-mode", + "--verbose", +} + + +def test_package_root_exports_remain_the_supported_python_api() -> None: + expect(tuple(r2morph.__all__) == _PUBLIC_EXPORTS) + + +def test_public_package_classes_resolve_from_the_package_root() -> None: + expect(all(callable(getattr(r2morph, name)) for name in _PUBLIC_EXPORTS[:-1])) + + +def test_cli_commands_remain_the_supported_top_level_commands() -> None: + command = get_command(app) + + expect(set(command.commands or {}) == _PUBLIC_COMMANDS) + + +def test_mutate_command_options_remain_compatible() -> None: + command = get_command(app).commands["mutate"] + long_options = { + option + for parameter in command.params + if hasattr(parameter, "opts") + for option in parameter.opts + if option.startswith("--") + } + + expect(long_options == _MUTATE_LONG_OPTIONS) + + +def test_console_script_remains_bound_to_the_supported_cli_app() -> None: + project = tomllib.loads((_ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] + + expect(project["scripts"]["r2morph"] == "r2morph.cli:app") diff --git a/tests/unit/test_release_contract.py b/tests/unit/test_release_contract.py index 11a68816..f81750c7 100644 --- a/tests/unit/test_release_contract.py +++ b/tests/unit/test_release_contract.py @@ -640,32 +640,32 @@ def test_release_contract_rejects_stale_vm_semantic_blocker_totals() -> None: def test_support_matrix_names_vm_resistance_gap_scope() -> None: matrix = json.loads((_ROOT / "docs" / "support-matrix.json").read_text(encoding="utf-8")) summary = matrix["matrix"]["summary"] - expected = ["human-adversarial-validation"] + expected: list[str] = [] + resolved = summary["vm_resistance_resolved_evidence"]["human-adversarial-validation"] expect( summary["vm_resistance_gap_scope"] == expected and summary["vm_resistance_evidence_blockers"]["vm_resistance_gap_scope"] == expected and summary["vm_resistance_blocker_totals"]["vm_resistance_gap_scope"] == len(expected) and summary["vm_resistance_blocker_totals"]["total_vm_resistance_blockers"] == len(expected) + and resolved["status"] == "approved" + and resolved["evidence_quality"] == "independent-human-review" ) -def test_support_matrix_names_vm_resistance_gap_evidence_without_signoff() -> None: +def test_support_matrix_names_resolved_vm_resistance_evidence_with_signoff() -> None: matrix = json.loads((_ROOT / "docs" / "support-matrix.json").read_text(encoding="utf-8")) summary = matrix["matrix"]["summary"] evidence = summary["vm_resistance_gap_evidence"] gap_scope = summary["vm_resistance_gap_scope"] - evidence_paths = { - item - for row in evidence.values() - for item in row["evidence"] - if isinstance(item, str) and not item.startswith("http") - } + resolved = summary["vm_resistance_resolved_evidence"]["human-adversarial-validation"] + evidence_paths = [item for item in resolved["evidence"] if isinstance(item, str) and not item.startswith("http")] expect( - sorted(evidence) == sorted(gap_scope) - and all(row["evidence_quality"] == "automated-adversarial-smoke" for gap, row in evidence.items()) - and all(row["status"] != "complete" for row in evidence.values()) + not gap_scope + and not evidence + and resolved["status"] == "approved" + and resolved["reviewed_commit"] == "a111f923df43a66826b71ab70db8a484532fbf3b" and all((_ROOT / path).exists() for path in evidence_paths) and summary["vm_resistance_blocker_totals"]["total_vm_resistance_blockers"] == len(gap_scope) ) @@ -673,10 +673,10 @@ def test_support_matrix_names_vm_resistance_gap_evidence_without_signoff() -> No def test_support_matrix_records_latest_vm_resistance_campaign() -> None: matrix = json.loads((_ROOT / "docs" / "support-matrix.json").read_text(encoding="utf-8")) - evidence = matrix["matrix"]["summary"]["vm_resistance_gap_evidence"] + evidence = matrix["matrix"]["summary"]["vm_resistance_resolved_evidence"] latest = "docs/protection-vm-resistance-2026-09-22-e3a491b3.json" - expect(all(latest in row["evidence"] for row in evidence.values())) + expect(latest in evidence["human-adversarial-validation"]["evidence"]) def test_release_contract_rejects_stale_vm_resistance_blocker_totals() -> None: @@ -692,15 +692,15 @@ def test_release_contract_rejects_stale_vm_resistance_blocker_totals() -> None: expect(rejected) -def test_release_contract_rejects_missing_vm_resistance_gap_evidence() -> None: +def test_release_contract_rejects_missing_resolved_vm_resistance_evidence() -> None: matrix = json.loads((_ROOT / "docs" / "support-matrix.json").read_text(encoding="utf-8")) - matrix["matrix"]["summary"]["vm_resistance_gap_evidence"].pop("human-adversarial-validation") + matrix["matrix"]["summary"]["vm_resistance_resolved_evidence"].pop("human-adversarial-validation") rejected = False try: _check_vm_resistance_gap_evidence(matrix) except ValueError as error: - rejected = "vm resistance gap evidence" in str(error) + rejected = "vm resistance" in str(error) expect(rejected) @@ -1222,12 +1222,12 @@ def test_independent_review_packet_lists_vm_resistance_reports() -> None: expect("protection-handler-clustering.json" in packet and "protection-bytecode-grammar.json" in packet) -def test_independent_review_packet_targets_current_main_head() -> None: +def test_independent_review_packet_binds_approval_to_reviewed_commit() -> None: packet = " ".join((_ROOT / "docs" / "independent-review-packet.md").read_text(encoding="utf-8").split()) expect( - "Review the exact pushed `main` HEAD when the human review begins" in packet - and "Reviewed commit (full 40-character SHA):" in packet + "Reviewed commit: `a111f923df43a66826b71ab70db8a484532fbf3b`" in packet + and "This approval applies to the reviewed commit only" in packet ) @@ -1409,7 +1409,7 @@ def test_release_contract_rejects_failed_independent_review_check() -> None: def test_release_contract_rejects_independent_review_without_vm_block() -> None: report = json.loads((_ROOT / "docs" / "independent-review.json").read_text(encoding="utf-8")) - report["release_decision"]["status"] = "ship" + report["release_decision"]["status"] = "block-vm-milestone" rejected = False try: @@ -1456,10 +1456,10 @@ def test_release_contract_requires_versioned_changelog_heading() -> None: expect(rejected) -def test_changelog_keeps_vm_milestone_blocker() -> None: +def test_changelog_records_independent_vm_review() -> None: changelog = (_ROOT / "CHANGELOG.md").read_text(encoding="utf-8") - expect("VM milestone blocked until external human review" in changelog) + expect("Independent human review approval is recorded" in changelog) def test_release_workflow_tests_installed_wheel_outside_checkout() -> None: @@ -1889,7 +1889,8 @@ def test_ci_cross_platform_smoke_runs_against_installed_wheel() -> None: and "Install radare2 (Windows)" in cross_platform_job and '"https://github.com/radareorg/radare2/releases/download/$version/$archiveName"' in cross_platform_job and '"913e7d95e7458226a5e783240877f2c1d405398ab38ba176ce9b12e046e31f34"' in cross_platform_job - and "Get-FileHash -Algorithm SHA256" in cross_platform_job + and '& ".github/scripts/download_verified_artifact.ps1" -Url $url -Output $archive -Sha256 $expectedSha256' + in cross_platform_job and "Expand-Archive -Path $archive -DestinationPath $destination -Force" in cross_platform_job and "$env:GITHUB_PATH" in cross_platform_job and 'run: python -m pip install "build>=1.2.0"' in cross_platform_job @@ -1942,6 +1943,26 @@ def test_ci_cross_platform_smoke_runs_against_installed_wheel() -> None: ) +def test_windows_radare2_workflows_use_bounded_verified_retries() -> None: + ci_workflow = (_ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8") + differential_workflow = (_ROOT / ".github" / "workflows" / "differential-corpus.yml").read_text(encoding="utf-8") + downloader = (_ROOT / ".github" / "scripts" / "download_verified_artifact.ps1").read_text(encoding="utf-8") + command = '& ".github/scripts/download_verified_artifact.ps1" -Url $url -Output $archive -Sha256 $expectedSha256' + + expect( + ci_workflow.count(command) == 1 + and differential_workflow.count(command) == 1 + and ' - ".github/scripts/download_verified_artifact.ps1"' in differential_workflow + and "$maximumAttempts = 3" in downloader + and "Invoke-WebRequest -Uri $Url -OutFile $partialOutput -TimeoutSec 120" in downloader + and "$statusCode -in @(408, 429)" in downloader + and "$statusCode -ge 500" in downloader + and "Start-Sleep -Seconds $attempt" in downloader + and "Get-FileHash -Algorithm SHA256 -Path $partialOutput" in downloader + and "Move-Item -LiteralPath $partialOutput -Destination $Output -Force" in downloader + ) + + def test_ci_broad_linux_suite_has_runtime_headroom() -> None: workflow = (_ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8") cross_platform_job = workflow.split(" cross-platform-tests:\n", 1)[1].split(" package-smoke:", 1)[0] diff --git a/tests/unit/test_release_contract_entrypoint.py b/tests/unit/test_release_contract_entrypoint.py new file mode 100644 index 00000000..1d297301 --- /dev/null +++ b/tests/unit/test_release_contract_entrypoint.py @@ -0,0 +1,22 @@ +"""Regression tests for the release-contract command entry point.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from tests.utils.assertions import expect +from tests.utils.process import run_command + +_ROOT = Path(__file__).resolve().parents[2] + + +def test_release_contract_script_entrypoint_loads_repository_root() -> None: + result = run_command( + [sys.executable, str(_ROOT / "scripts" / "check_release_contract.py")], + cwd=_ROOT, + text=True, + timeout=30, + ) + + expect(result.returncode == 0 and "release contract valid:" in result.stdout, result.stderr) diff --git a/tests/unit/test_release_workflow_contract.py b/tests/unit/test_release_workflow_contract.py new file mode 100644 index 00000000..a9ac0a38 --- /dev/null +++ b/tests/unit/test_release_workflow_contract.py @@ -0,0 +1,40 @@ +"""Regression contracts for release workflow bootstrap ordering.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml + +from tests.utils.assertions import expect + +_ROOT = Path(__file__).resolve().parents[2] +_EXPECTED_PUBLIC_SURFACE_CONTRACT_WORKFLOW_OCCURRENCES = 2 + + +def test_validate_release_installs_checker_dependencies_before_running_contract() -> None: + workflow: dict[str, Any] = yaml.safe_load((_ROOT / ".github" / "workflows" / "release.yml").read_text()) + steps = workflow["jobs"]["validate-release"]["steps"] + dependency_step = next( + (index for index, step in enumerate(steps) if step.get("name") == "Install release dependencies"), None + ) + contract_step = next( + (index for index, step in enumerate(steps) if step.get("name") == "Validate support contract"), None + ) + + expect( + dependency_step is not None + and contract_step is not None + and dependency_step < contract_step + and "pip install -r requirements.txt" in steps[dependency_step]["run"] + ) + + +def test_installed_wheel_jobs_run_the_public_api_and_cli_contract() -> None: + workflow = (_ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8") + + expect( + workflow.count("tests/unit/test_public_surface_contract.py") + == _EXPECTED_PUBLIC_SURFACE_CONTRACT_WORKFLOW_OCCURRENCES + ) diff --git a/tests/unit/test_sarif_formatter.py b/tests/unit/test_sarif_formatter.py index de7807ad..7236fc6b 100644 --- a/tests/unit/test_sarif_formatter.py +++ b/tests/unit/test_sarif_formatter.py @@ -393,6 +393,24 @@ def test_artifacts_in_report(self, formatter): expect(artifacts is not None) expect(len(artifacts) == _EXPECTED_LEN_ARTIFACTS_2) + def test_sarif_artifacts_exclude_parent_paths(self, formatter): + report = formatter.format( + ReportData( + binary_path="/Users/analyst/private/input.exe", + output_path=r"C:\Users\analyst\private\output.exe", + ) + ) + + artifacts = report.runs[0].artifacts + expect( + artifacts is not None and [artifact.location.uri for artifact in artifacts] == ["input.exe", "output.exe"] + ) + + def test_sarif_report_omits_host_source_root(self, formatter): + report = formatter.to_json(ReportData(binary_path="/Users/analyst/private/input.exe")) + + expect("originalUriBaseIds" not in report) + def test_properties_in_result(self, formatter, sample_mutation): report_data = ReportData( binary_path="test.exe", diff --git a/tests/unit/test_semantic_symbolic_contract.py b/tests/unit/test_semantic_symbolic_contract.py index f2213693..183690e5 100644 --- a/tests/unit/test_semantic_symbolic_contract.py +++ b/tests/unit/test_semantic_symbolic_contract.py @@ -1,7 +1,5 @@ from types import SimpleNamespace -import claripy - from r2morph.validation.semantic_symbolic import ( create_symbolic_state, default_observables, @@ -28,6 +26,6 @@ def blank_state(self, *, addr: int) -> SimpleNamespace: created = create_symbolic_state(project, 0x1000, 64, ["rax", "eflags"]) expect(not (created is not state)) - expect(claripy.is_true(state.regs.rsp == _EXPECTED_STATE_REGS_RSP_1048576)) - expect(claripy.is_true(state.regs.rbp == _EXPECTED_STATE_REGS_RBP_1048576)) + expect(state.regs.rsp.op == "BVV" and state.regs.rsp.args[0] == _EXPECTED_STATE_REGS_RSP_1048576) + expect(state.regs.rbp.op == "BVV" and state.regs.rbp.args[0] == _EXPECTED_STATE_REGS_RBP_1048576) expect(next(iter(state.regs.rax.variables)).startswith("rax_1000_")) diff --git a/tests/unit/test_support_matrix_contract.py b/tests/unit/test_support_matrix_contract.py index aba6f2dd..c84462a2 100644 --- a/tests/unit/test_support_matrix_contract.py +++ b/tests/unit/test_support_matrix_contract.py @@ -1,5 +1,6 @@ """Regression contract for exhaustive pass support cells.""" +import ast import json from pathlib import Path @@ -9,11 +10,14 @@ _README = Path(__file__).resolve().parents[2] / "README.md" _PASS_MATURITY = Path(__file__).resolve().parents[2] / "docs" / "pass-maturity.md" _MATRIX = Path(__file__).resolve().parents[2] / "docs" / "support-matrix.json" -_EXPECTED_EVIDENCED_CELLS = 54 -_EXPECTED_NOT_SUPPORTED_CELLS = 210 -_EXPECTED_NON_OFFICIAL_EVIDENCED_CELLS = 32 -_EXPECTED_NON_OFFICIAL_NOT_SUPPORTED_CELLS = 210 -_EXPECTED_NON_OFFICIAL_MISSING_EVIDENCE_CELLS = 210 +_EXPECTED_EVIDENCED_CELLS = 57 +_EXPECTED_NOT_SUPPORTED_CELLS = 207 +_EXPECTED_DECLARED_SUPPORT_CELLS = 57 +_EXPECTED_UNCLAIMED_CELLS = 207 +_EXPECTED_TARGET_SUPPORT_STATUS_COUNTS = {"experimental": 45, "preview": 7, "supported": 5} +_EXPECTED_NON_OFFICIAL_EVIDENCED_CELLS = 35 +_EXPECTED_NON_OFFICIAL_NOT_SUPPORTED_CELLS = 207 +_EXPECTED_NON_OFFICIAL_MISSING_EVIDENCE_CELLS = 207 _EXPECTED_STABILITY_COUNTS = {"experimental": 17, "tier-1": 5} _EXPECTED_MATURITY_PROFILE_COUNTS = { "anti-disassembly-instruction-catalogued": 1, @@ -68,6 +72,9 @@ def test_support_matrix_summarizes_parity_gaps() -> None: expect( summary["evidenced_cells"] == _EXPECTED_EVIDENCED_CELLS and summary["not_supported_cells"] == _EXPECTED_NOT_SUPPORTED_CELLS + and summary["declared_support_cells"] == _EXPECTED_DECLARED_SUPPORT_CELLS + and summary["unclaimed_cells"] == _EXPECTED_UNCLAIMED_CELLS + and summary["target_support_status_counts"] == _EXPECTED_TARGET_SUPPORT_STATUS_COUNTS and summary["non_official_evidenced_cells"] == _EXPECTED_NON_OFFICIAL_EVIDENCED_CELLS and summary["non_official_not_supported_cells"] == _EXPECTED_NON_OFFICIAL_NOT_SUPPORTED_CELLS and parity_totals["non_official_missing_evidence_cells"] == _EXPECTED_NON_OFFICIAL_MISSING_EVIDENCE_CELLS @@ -122,6 +129,8 @@ def test_pass_maturity_keeps_preview_targets_out_of_parity_claims() -> None: expect( "Preview PE, Mach-O, ARM, and AArch64 evidence does not imply parity" in contract + and "`target_cells` is the canonical support contract" in contract + and "must not be cross-multiplied" in contract and "preview coverage cannot look equivalent to the supported baseline" in contract and str(summary["non_official_evidenced_cells"]) in contract and str(summary["non_official_not_supported_cells"]) in contract @@ -186,9 +195,16 @@ def test_support_matrix_honors_explicit_evidence_cells() -> None: "passes": [ { "name": "nop", - "formats": [], - "architectures": [], - "evidence_cells": [{"format": "PE", "architecture": "AArch64"}], + "formats": ["PE"], + "architectures": ["AArch64"], + "target_cells": [{"format": "PE", "architecture": "AArch64", "support_status": "experimental"}], + "evidence_cells": [ + { + "format": "PE", + "architecture": "AArch64", + "evidence": ["tests/integration/test_pe_aarch64.py::test_real_execution"], + } + ], "evidence": ["tests/integration"], } ], @@ -197,15 +213,121 @@ def test_support_matrix_honors_explicit_evidence_cells() -> None: expect(build_matrix(document)["cells"][0]["status"] == "evidenced") +def test_support_matrix_does_not_expand_target_cells_cartesianly() -> None: + document = { + "formats": {"ELF": "official", "PE": "preview"}, + "architectures": {"x86-64": "official", "AArch64": "experimental"}, + "official_target": {"format": "ELF", "architecture": "x86-64"}, + "passes": [ + { + "name": "nop", + "stability": "tier-1", + "formats": ["ELF", "PE"], + "architectures": ["x86-64", "AArch64"], + "target_cells": [ + {"format": "ELF", "architecture": "x86-64", "support_status": "supported"}, + {"format": "PE", "architecture": "x86-64", "support_status": "preview"}, + {"format": "ELF", "architecture": "AArch64", "support_status": "experimental"}, + ], + "evidence_cells": [ + { + "format": "ELF", + "architecture": "x86-64", + "evidence": ["tests/integration/test_elf_nop.py::test_nop"], + }, + { + "format": "PE", + "architecture": "x86-64", + "evidence": ["tests/integration/test_pe_nop.py::test_nop"], + }, + { + "format": "ELF", + "architecture": "AArch64", + "evidence": ["tests/integration/test_arm64_nop.py::test_nop"], + }, + ], + "evidence": ["tests/integration"], + } + ], + } + + cells = build_matrix(document)["cells"] + pe_aarch64 = next(cell for cell in cells if cell["format"] == "PE" and cell["architecture"] == "AArch64") + + expect( + not pe_aarch64["declared_support"] + and pe_aarch64["support_status"] is None + and pe_aarch64["status"] == "not-supported" + ) + + +def test_support_matrix_uses_target_specific_evidence_without_inheriting_other_targets() -> None: + document = { + "formats": {"ELF": "official", "PE": "preview"}, + "architectures": {"x86-64": "official"}, + "official_target": {"format": "ELF", "architecture": "x86-64"}, + "passes": [ + { + "name": "nop", + "stability": "tier-1", + "formats": ["ELF", "PE"], + "architectures": ["x86-64"], + "target_cells": [ + {"format": "ELF", "architecture": "x86-64", "support_status": "supported"}, + {"format": "PE", "architecture": "x86-64", "support_status": "preview"}, + ], + "evidence": ["tests/integration/test_linux_nop.py::test_linux_execution"], + "evidence_cells": [ + { + "format": "PE", + "architecture": "x86-64", + "evidence": ["tests/integration/test_pe_nop.py::test_pe_execution"], + } + ], + } + ], + } + + pe_cell = next(cell for cell in build_matrix(document)["cells"] if cell["format"] == "PE") + + expect(pe_cell["evidence"] == ["tests/integration/test_pe_nop.py::test_pe_execution"]) + + +def test_support_matrix_preview_evidence_references_real_test_cases() -> None: + document = json.loads(_MATRIX.read_text(encoding="utf-8")) + preview_cells = [ + cell + for cell in document["matrix"]["cells"] + if cell["status"] == "evidenced" and (cell["format"] != "ELF" or cell["architecture"] != "x86-64") + ] + missing_cases = [] + for cell in preview_cells: + for reference in cell["evidence"]: + path_text, case_name = reference.split("::", maxsplit=1) + source = (_MATRIX.parents[1] / path_text).read_text(encoding="utf-8") + definitions = { + node.name + for node in ast.walk(ast.parse(source)) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + if case_name not in definitions: + missing_cases.append(reference) + + expect(not missing_cases) + + def test_support_matrix_does_not_turn_declared_support_into_evidence() -> None: document = { "formats": {"ELF": "official"}, "architectures": {"x86-64": "official"}, + "official_target": {"format": "ELF", "architecture": "x86-64"}, "passes": [ { "name": "nop", + "stability": "tier-1", "formats": ["ELF"], "architectures": ["x86-64"], + "target_cells": [{"format": "ELF", "architecture": "x86-64", "support_status": "supported"}], "evidence_cells": [], "evidence": ["tests/integration"], } @@ -222,7 +344,7 @@ def test_support_matrix_keeps_non_official_targets_out_of_supported_status() -> cells = build_matrix(document)["cells"] non_official_cells = [cell for cell in cells if cell["format"] != "ELF" or cell["architecture"] != "x86-64"] - expect(non_official_cells and all(cell["status"] != "supported" for cell in non_official_cells)) + expect(non_official_cells and all(cell["support_status"] != "supported" for cell in non_official_cells)) def test_support_matrix_records_preview_cells_backed_by_native_smoke() -> None: @@ -231,12 +353,58 @@ def test_support_matrix_records_preview_cells_backed_by_native_smoke() -> None: expect( all( - cell in cells_by_pass[pass_name] + any( + evidence_cell["format"] == cell["format"] and evidence_cell["architecture"] == cell["architecture"] + for evidence_cell in cells_by_pass[pass_name] + ) for pass_name, cell in ( ("nop", {"format": "PE", "architecture": "x86-64"}), + ("block-reordering", {"format": "PE", "architecture": "x86-64"}), ("substitute", {"format": "Mach-O", "architecture": "x86-64"}), ("register", {"format": "Mach-O", "architecture": "x86-64"}), ("instruction-expansion", {"format": "Mach-O", "architecture": "x86-64"}), ) ) ) + + +def test_support_matrix_records_macho_block_reordering_as_one_preview_case() -> None: + document = json.loads(_MATRIX.read_text(encoding="utf-8")) + entry = next(item for item in document["passes"] if item["name"] == "block-reordering") + cell = next( + cell + for cell in build_matrix(document)["cells"] + if cell["pass"] == entry["name"] and cell["format"] == "Mach-O" and cell["architecture"] == "x86-64" + ) + + expect( + entry["stability"] == "tier-1" + and "Mach-O" in entry["formats"] + and cell["support_status"] == "experimental" + and cell["status"] == "evidenced" + and cell["evidence"] + == [ + "tests/integration/test_mutations_x86_compiled_binary.py::" + "test_x86_block_reordering_preserves_native_exit_code" + ] + ) + + +def test_support_matrix_records_pe_block_reordering_as_preview_evidence() -> None: + document = json.loads(_MATRIX.read_text(encoding="utf-8")) + entry = next(item for item in document["passes"] if item["name"] == "block-reordering") + cell = next( + cell + for cell in build_matrix(document)["cells"] + if cell["pass"] == entry["name"] and cell["format"] == "PE" and cell["architecture"] == "x86-64" + ) + + expect( + entry["stability"] == "tier-1" + and cell["support_status"] == "preview" + and cell["status"] == "evidenced" + and cell["evidence"] + == [ + "tests/integration/test_block_reordering_pe_real.py::test_pe_x86_64_block_reordering_preserves_native_observables" + ] + ) diff --git a/tests/unit/test_syntia_integration_features.py b/tests/unit/test_syntia_integration_features.py index 4611bec5..101e4c04 100644 --- a/tests/unit/test_syntia_integration_features.py +++ b/tests/unit/test_syntia_integration_features.py @@ -1,6 +1,10 @@ +import json +from io import StringIO + import pytest from r2morph.analysis.symbolic.syntia_integration import InstructionSemantics, SyntiaFramework, VMHandlerSemantics +from r2morph.utils.logging import setup_logging from tests.utils.assertions import expect _EXPECTED_FRAMEWORK_EVALUATE_EXPRESSION_X_X_0_4294967295 = 0xFFFFFFFF @@ -11,6 +15,17 @@ pytestmark = [pytest.mark.experimental] +def test_syntia_expression_diagnostic_is_debug_only(): + log_stream = StringIO() + setup_logging("DEBUG", stream=log_stream, environ={"R2MORPH_LOG_FORMAT": "json"}) + + SyntiaFramework().simplify_mba_with_syntia("x + x", variables={"x"}) + + events = [json.loads(line) for line in log_stream.getvalue().splitlines()] + detail = next(event for event in events if event["message"] == "Simplifying MBA expression: x + x") + expect(detail["level"] == "DEBUG") + + def test_syntia_vm_handler_analysis_and_exports(tmp_path): framework = SyntiaFramework() diff --git a/tests/unit/test_unit_test_job_timeout_contract.py b/tests/unit/test_unit_test_job_timeout_contract.py new file mode 100644 index 00000000..b9adab50 --- /dev/null +++ b/tests/unit/test_unit_test_job_timeout_contract.py @@ -0,0 +1,20 @@ +"""Regression contract for bounded unit-test CI execution.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml + +from tests.utils.assertions import expect + +_ROOT = Path(__file__).resolve().parents[2] +_UNIT_TEST_TIMEOUT_MINUTES = 30 + + +def test_unit_test_job_has_thirty_minute_timeout() -> None: + workflow: dict[str, Any] = yaml.safe_load((_ROOT / ".github" / "workflows" / "ci.yml").read_text()) + job = workflow["jobs"]["unit-tests"] + + expect(job.get("timeout-minutes") == _UNIT_TEST_TIMEOUT_MINUTES)