diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore new file mode 100644 index 00000000..09e58b73 --- /dev/null +++ b/benchmarks/.gitignore @@ -0,0 +1,5 @@ +elm-stuff/ +elm_home/ +elm.js +index.html +results/ diff --git a/benchmarks/Main.elm b/benchmarks/Main.elm index 92ef6aec..a9f7557b 100644 --- a/benchmarks/Main.elm +++ b/benchmarks/Main.elm @@ -1,80 +1,240 @@ -module Main exposing (main) +port module Main exposing (main) -import Benchmark exposing (..) -import Benchmark.Runner as Runner -import Expect exposing (Expectation) +{-| Throughput benchmarks for elm-test. + +Where `f-metric/` asks "is elm-test good at finding bugs", this asks "how fast +does it get through a test". The two answer different questions, and the F-metric +harness structurally cannot answer this one: its whole corpus is failing tests, +so it never measures the case that dominates a real suite -- a fuzz test that +passes and therefore runs its full budget. + +That's also the case where exhaustive checking should pay off by stopping early: +`Fuzz.bool` given `--fuzz 100` has two possible inputs and ought to cost two +runs, not a hundred. + +Headless rather than browser-based, so it runs in CI and so numbers can be +produced without a human clicking anything. That means it no longer uses +elm-explorations/benchmark, whose runner is a browser program; the statistics are +simple enough to do in `runner.mjs`. + +Uses the public `Test.Runner` API. Its `run : () -> List Expectation` is a plain +function, which is what makes it benchmarkable -- `Test.RunnerV2.runFuzzTest` +returns a `Task`, and a Task can't be timed from inside Elm. + +-} + +import Array exposing (Array) +import Json.Decode as Decode +import Json.Encode as Encode +import Platform import Random import Snippets -import Test.Internal exposing (Test(..)) +import Test exposing (Test) +import Test.Runner -main : Runner.BenchmarkProgram +port toJs : Encode.Value -> Cmd msg + + +port fromJs : (Decode.Value -> msg) -> Sub msg + + +main : Program () () Msg main = - Runner.program suite - - -suite : Benchmark -suite = - describe "Fuzz" - [ describe "int" - [ benchmark "generating" (benchTest Snippets.intPass) - , benchmark "simplifying" (benchTest Snippets.intFail) - ] - , describe "intRange" - [ benchmark "generating" (benchTest Snippets.intRangePass) - , benchmark "simplifying" (benchTest Snippets.intRangeFail) - ] - , describe "string" - [ benchmark "generating" (benchTest Snippets.stringPass) - , benchmark "simplifying" (benchTest Snippets.stringFail) - ] - , describe "float" - [ benchmark "generating" (benchTest Snippets.floatPass) - , benchmark "simplifying" (benchTest Snippets.floatFail) - ] - , describe "bool" - [ benchmark "generating" (benchTest Snippets.boolPass) - , benchmark "simplifying" (benchTest Snippets.boolFail) - ] - , describe "char" - [ benchmark "generating" (benchTest Snippets.charPass) - , benchmark "simplifying" (benchTest Snippets.charFail) - ] - , describe "list of int" - [ benchmark "generating" (benchTest Snippets.listIntPass) - , benchmark "simplifying" (benchTest Snippets.listIntFail) - ] - , describe "maybe of int" - [ benchmark "generating" (benchTest Snippets.maybeIntPass) - , benchmark "simplifying" (benchTest Snippets.maybeIntFail) - ] - , describe "result of string and int" - [ benchmark "generating" (benchTest Snippets.resultPass) - , benchmark "simplifying" (benchTest Snippets.resultFail) - ] - , describe "map" - [ benchmark "generating" (benchTest Snippets.mapPass) - , benchmark "simplifying" (benchTest Snippets.mapFail) - ] - , describe "andMap" - [ benchmark "generating" (benchTest Snippets.andMapPass) - , benchmark "simplifying" (benchTest Snippets.andMapFail) - ] - , describe "map5" - [ benchmark "generating" (benchTest Snippets.map5Pass) - , benchmark "simplifying" (benchTest Snippets.map5Fail) - ] - ] - - -benchTest : Test -> (() -> List Expectation) -benchTest test = - case test of - ElmTestVariant__FuzzTest fn -> - \_ -> fn (Random.initialSeed 0) 10 - - ElmTestVariant__Labeled _ test_ -> - benchTest test_ - - test_ -> - Debug.todo <| "No support for benchmarking this type of test: " ++ Debug.toString test_ + Platform.worker + { init = \_ -> ( (), Cmd.none ) + , update = update + , subscriptions = \_ -> fromJs FromJs + } + + +type Msg + = FromJs Decode.Value + + +type Request + = Manifest + | Run RunParams + + +type alias RunParams = + { index : Int + , runs : Int + , seed : Int + , iterations : Int + } + + +update : Msg -> () -> ( (), Cmd Msg ) +update (FromJs raw) model = + case Decode.decodeValue requestDecoder raw of + Err error -> + ( model + , toJs + (Encode.object + [ ( "tag", Encode.string "error" ) + , ( "error", Encode.string (Decode.errorToString error) ) + ] + ) + ) + + Ok Manifest -> + ( model + , toJs + (Encode.object + [ ( "tag", Encode.string "manifest" ) + , ( "benchmarks" + , Encode.list + (\( name, passes, _ ) -> + Encode.object + [ ( "name", Encode.string name ) + + {- runs/sec is only meaningful for a test that + actually executes its whole budget. A failing + test stops at the first value, so dividing by + `runs` would overstate it by orders of + magnitude. + -} + , ( "passes", Encode.bool passes ) + ] + ) + benchmarks + ) + ] + ) + ) + + Ok (Run params) -> + case Array.get params.index (Array.fromList benchmarks) of + Nothing -> + ( model + , toJs + (Encode.object + [ ( "tag", Encode.string "error" ) + , ( "error", Encode.string "no benchmark at that index" ) + ] + ) + ) + + Just ( _, _, test ) -> + ( model + , toJs + (Encode.object + [ ( "tag", Encode.string "done" ) + + {- The count exists to force the work. Without + consuming the expectations there'd be nothing + stopping the whole loop being optimized away. + -} + , ( "checksum", Encode.int (repeat params test) ) + ] + ) + ) + + +requestDecoder : Decode.Decoder Request +requestDecoder = + Decode.field "tag" Decode.string + |> Decode.andThen + (\tag -> + case tag of + "manifest" -> + Decode.succeed Manifest + + "run" -> + Decode.map4 RunParams + (Decode.field "index" Decode.int) + (Decode.field "runs" Decode.int) + (Decode.field "seed" Decode.int) + (Decode.field "iterations" Decode.int) + |> Decode.map Run + + _ -> + Decode.fail ("unknown tag: " ++ tag) + ) + + +{-| Run the whole test `iterations` times, returning the number of expectations +produced so the work can't be elided. + +Each iteration rebuilds the runners from scratch, which is what a real runner +does per test, and keeps one iteration from warming state for the next. + +-} +repeat : RunParams -> Test -> Int +repeat params test = + repeatHelp params test params.iterations 0 + + +repeatHelp : RunParams -> Test -> Int -> Int -> Int +repeatHelp params test remaining acc = + if remaining <= 0 then + acc + + else + let + expectations : Int + expectations = + case Test.Runner.fromTest params.runs (Random.initialSeed params.seed) test of + Test.Runner.Plain runners -> + List.foldl (\runner sum -> sum + List.length (runner.run ())) 0 runners + + _ -> + 0 + in + repeatHelp params test (remaining - 1) (acc + expectations) + + + +-- BENCHMARKS + + +{-| Name, and the test to run. + +The `(passes)` cases are the interesting ones for throughput: they run the full +`runs` budget. The `simplify/` ones stop at the first value and then simplify, so +they measure simplification cost instead. + +-} +benchmarks : List ( String, Bool, Test ) +benchmarks = + [ -- Small enough to enumerate exhaustively. If exhaustive checking lands, + -- these should collapse to a couple of runs regardless of the budget. + ( "small/unit", True, Snippets.unitPass ) + , ( "small/bool", True, Snippets.boolPass ) + , ( "small/order", True, Snippets.orderPass ) + , ( "small/intRange-0-20", True, Snippets.intRange0To20Pass ) + , ( "small/pair-bool", True, Snippets.pairBoolPass ) + , ( "small/maybe-bool", True, Snippets.maybeBoolPass ) + , ( "small/oneOfValues", True, Snippets.oneOfValuesPass ) + + -- Mid-size: finite, but not trivially so. These decide where a budget + -- boundary should sit. + , ( "mid/intRange-0-1000", True, Snippets.intRange0To1000Pass ) + , ( "mid/pair-intRange-0-30", True, Snippets.pairIntRange0To30Pass ) + + -- Unbounded. These must not regress: they're where an enumeration probe or + -- deduplication bookkeeping is pure overhead. + , ( "large/int", True, Snippets.intPass ) + , ( "large/float", True, Snippets.floatPass ) + , ( "large/string", True, Snippets.stringPass ) + , ( "large/list-int", True, Snippets.listIntPass ) + , ( "large/record-map5", True, Snippets.map5Pass ) + + {- Same fuzzers, but with a test body that does real work. Coverage tracking's + overhead is mostly per-run and fixed, so a heavier body dilutes it; early + termination skips the body too, so a heavier body makes that win larger. + -} + , ( "heavy/bool", True, Snippets.boolHeavyPass ) + , ( "heavy/pair-bool", True, Snippets.pairBoolHeavyPass ) + , ( "heavy/string", True, Snippets.stringHeavyPass ) + , ( "heavy/list-int", True, Snippets.listIntHeavyPass ) + + -- Rejection path. + , ( "filter/even", True, Snippets.filterPass ) + + -- Simplification cost, for contrast with the passing cases. + , ( "simplify/int", False, Snippets.intFail ) + , ( "simplify/string", False, Snippets.stringFail ) + , ( "simplify/list-int", False, Snippets.listIntFail ) + ] diff --git a/benchmarks/README.md b/benchmarks/README.md index f4f0aa53..7b099800 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,34 +1,125 @@ -# Benchmarks for elm-test +# Throughput benchmarks for elm-test -These are some benchmarks of the elm-test library built using the excellent [elm-benchmark](https://github.com/BrianHicks/elm-benchmark). +How fast elm-test gets through a test. For whether it's any *good* at finding +bugs, see `f-metric/`. -## How to run +The two are complementary and neither substitutes for the other. The F-metric +corpus is entirely *failing* tests, so it structurally cannot measure the case +that dominates a real suite: a fuzz test that passes and therefore runs its whole +budget. That's also where exhaustive checking should pay off, by stopping as soon +as the input domain is covered. + +## Running ```sh -cd ./benchmarks -elm make Main.elm -open index.html +cd benchmarks +./build.sh +node runner.mjs --runs 100 +``` + +`./build.sh` copies `../src` into a local `ELM_HOME` and compiles against it, so +you always measure the working copy. + +### Comparing two implementations + +```sh +# on master +./build.sh && node runner.mjs --runs 1000 --label before --out /tmp/before.json + +# on your branch +./build.sh && node runner.mjs --runs 1000 --label after --compare /tmp/before.json +``` + +Comparison is on `ms/test`, which is defined for passing and failing benchmarks +alike, and reports a geometric mean so one slow benchmark can't dominate. Use the +same `--runs` on both sides. + +Add `--markdown` for tables you can paste into a PR. + +## Reading the output + +``` +benchmark runs/sec ms/test iters spread +small/bool 7.48M 0.0134 4096 14% +simplify/int n/a 0.0070 8192 13% ``` -## How to use -These benchmarks can help get an idea of the performance impact of a change in the elm-test code. -Beware however that a fifty percent performance increase in these benchmarks will most likely not translate to a fifty percent faster tests for users. -In real word scenario's the execution of the test body will have a significant impact on the running time of the test suite, an aspect we're not testing here because it's different for every test suite. -To get a feeling for the impact your change has on actual test run times try running some real test suites with and without your changes. - -## Benchmarking complete test suites -These are some examples of test suites that contain a lot of fuzzer tests: -- [elm-benchmark](https://github.com/BrianHicks/elm-benchmark) -- [elm-nonempty-list](https://github.com/mgold/elm-nonempty-list) -- [json-elm-schema](https://github.com/NoRedInk/json-elm-schema) - -A tool you can use for benchmarking the suite is [bench](https://github.com/Gabriel439/bench). - -To run the tests using your modified code (this only works if your modified version is backwards compatible with the version of elm-test currenlty in use by the test suite): -- In your test suite directories `elm-package.json`: - - Remove the dependency on `elm-test`. - - Add dependecies of `elm-test` as dependencies of the test suite itself. - - Add the path to your changed elm-test src directory to your `source-directories`. - It will be something like `//elm-test/src`. -- Run `elm-test` once to trigger compilation. -- Now run `elm-test` with your benchmarking tool. +- `runs/sec` — fuzz runs per second, so it stays comparable across `--runs` + settings. **`n/a` for failing benchmarks**: they stop at the first value + instead of executing the whole budget, so dividing by `--runs` would overstate + them by orders of magnitude. Read `ms/test` for those. +- `ms/test` — wall time for one whole test, budget included. +- `iters` — how many times the test was run per timed repetition, grown + automatically until a repetition clears `--min-ms` so the timer isn't what's + being measured. Doubles as warmup. +- `spread` — max-to-min across repetitions, as a percentage of the median. + Currently 1–20%; treat anything under ~1.2x as noise. + +## The benchmark groups + +| group | why | +| :--- | :--- | +| `small/` | domains small enough to enumerate exhaustively. **The early-termination opportunity.** | +| `mid/` | finite but not trivially so — these decide where a budget boundary should sit | +| `large/` | unbounded domains. **The regression guard**: an enumeration probe or deduplication bookkeeping is pure overhead here | +| `filter/` | the rejection path, which deduplication and enumeration both have to handle | +| `simplify/` | failing tests, so simplification cost rather than generation throughput | + +## The measurement that motivates exhaustive checking + +Cost scales linearly with the budget even when the domain has a single element: + +| benchmark | domain size | ms/test @ `--runs 100` | @ `--runs 1000` | scaling | +| :--- | ---: | ---: | ---: | ---: | +| `small/unit` | 1 | 0.0069 | 0.0492 | 7.1x | +| `small/bool` | 2 | 0.0134 | 0.1077 | 8.0x | +| `small/order` | 3 | 0.0207 | 0.1972 | 9.5x | +| `small/pair-bool` | 4 | 0.0243 | 0.2094 | 8.6x | +| `small/intRange-0-20` | 21 | 0.0166 | 0.1341 | 8.1x | +| `large/string` | unbounded | 0.3016 | 2.9618 | 9.8x | +| `large/list-int` | unbounded | 0.7599 | 8.2087 | 10.8x | + +`Fuzz.unit` has exactly one possible input and we test it a thousand times. The +recoverable factor is `runs / domainSize`, so it grows without bound in whatever +`--fuzz` the user passes — and unlike deduplicating already-tested runs, stopping +early avoids *generating* the redundant values too, which is where the cost +actually is. + +The `large/` rows scale linearly for real reasons and have nothing to recover. +They're the rows to watch for regressions. + +## Notes on the implementation + +- **Headless, not browser-based.** This used to be an + [elm-explorations/benchmark](https://github.com/elm-explorations/benchmark) + browser program, which meant it ran in CI never and produced numbers only when + a human clicked something. The statistics here are simple enough to do in + `runner.mjs`. +- **No library internals.** It used to destructure + `Test.Internal.ElmTestVariant__FuzzTest`, which broke when that variant changed + shape. It now uses the public `Test.Runner.fromTest`, whose + `run : () -> List Expectation` is a plain function — which is what makes it + benchmarkable, since `Test.RunnerV2.runFuzzTest` returns a `Task` and a Task + can't be timed from inside Elm. +- **`source-directories` can't be used to pull in `../src`.** Applications aren't + allowed to import `Elm.Kernel.*` modules, so that approach stopped compiling as + soon as `RandomRun` moved into the kernel. Hence the local `ELM_HOME`, the same + trick `tests/` and `f-metric/` use. +- `build.sh` clears `elm-stuff`, because Elm caches compiled dependency artifacts + per package *version* and the version never changes as `../src` is edited. + Without that, a build silently reuses a stale copy of the library. + +## Benchmarking a real test suite + +These microbenchmarks won't tell you what a fifty percent improvement here does +to a real suite, where the test bodies do real work. To check that, point a real +suite at your modified `src`: + +- In the suite's `elm.json`, remove the `elm-explorations/test` dependency, add + elm-test's own dependencies, and add the path to your `src` to + `source-directories`. +- Run the suite once to compile, then time it. + +The library's own `tests/` suite works as an independent workload too: build once +with `bash tests/run-tests.sh`, then time `node tests/elm.js` repeatedly. Don't +use `NOCLEANUP=1` for that — same stale-artifact trap as above. diff --git a/benchmarks/Snippets.elm b/benchmarks/Snippets.elm index 6b885059..3be38bdf 100644 --- a/benchmarks/Snippets.elm +++ b/benchmarks/Snippets.elm @@ -1,7 +1,9 @@ module Snippets exposing (..) +import Dict import Expect exposing (Expectation) import Fuzz exposing (Fuzzer) +import Set import Test exposing (Test, fuzz) @@ -208,3 +210,132 @@ sequence fuzzers = (Fuzz.map2 (::)) (Fuzz.constant []) fuzzers + + + +{- Passing tests over small, exhaustible domains. These are the ones exhaustive + checking should be able to finish early: the whole input space fits in far + fewer values than a typical `runs` budget. +-} + + +unitPass : Test +unitPass = + fuzz "(passes) unit" Fuzz.unit <| + \_ -> Expect.pass + + +orderPass : Test +orderPass = + fuzz "(passes) order" Fuzz.order <| + \_ -> Expect.pass + + +intRange0To20Pass : Test +intRange0To20Pass = + fuzz "(passes) intRange 0 20" (Fuzz.intRange 0 20) <| + \_ -> Expect.pass + + +pairBoolPass : Test +pairBoolPass = + fuzz "(passes) pair of bools" (Fuzz.pair Fuzz.bool Fuzz.bool) <| + \_ -> Expect.pass + + +maybeBoolPass : Test +maybeBoolPass = + fuzz "(passes) maybe bool" (Fuzz.maybe Fuzz.bool) <| + \_ -> Expect.pass + + +oneOfValuesPass : Test +oneOfValuesPass = + fuzz "(passes) oneOfValues" (Fuzz.oneOfValues [ 1, 2, 3, 4, 5 ]) <| + \_ -> Expect.pass + + + +{- Finite, but big enough that exhausting it is a real decision rather than a + freebie. +-} + + +intRange0To1000Pass : Test +intRange0To1000Pass = + fuzz "(passes) intRange 0 1000" (Fuzz.intRange 0 1000) <| + \_ -> Expect.pass + + +pairIntRange0To30Pass : Test +pairIntRange0To30Pass = + fuzz "(passes) pair of intRange 0 30" + (Fuzz.pair (Fuzz.intRange 0 30) (Fuzz.intRange 0 30)) + <| + \_ -> Expect.pass + + + +{- Exercises the rejection path, which deduplication and enumeration both have + to handle. Kept generous: `Fuzz.filter` gives up after 16 consecutive + rejections and fails the test, and over thousands of runs even a mildly + selective predicate hits that. +-} + + +filterPass : Test +filterPass = + fuzz "(passes) filtered intRange" + (Fuzz.intRange 0 100 |> Fuzz.filter (\n -> modBy 4 n /= 0)) + <| + \_ -> Expect.pass + + + +{- Varying how much work the *test body* does, holding the fuzzer fixed. + + The overhead of coverage tracking is largely per-run and fixed, so a heavier + body should dilute it. Early termination works the other way: skipping a run + skips its body too, so the heavier the body the bigger the saving. These pairs + measure both effects, which is what a real test suite with substantial + assertions would exercise. +-} + + +{-| An assertion with real work in it: round-trip the list through a Dict and +compare, which allocates and compares structures rather than checking a tag. +-} +heavyAssertion : List Int -> Expect.Expectation +heavyAssertion xs = + xs + |> List.map (\x -> ( x, String.fromInt x )) + |> Dict.fromList + |> Dict.toList + |> List.map Tuple.first + |> Expect.equal (xs |> Set.fromList |> Set.toList) + + +listIntHeavyPass : Test +listIntHeavyPass = + fuzz "(passes) list of int, heavy assertion" (Fuzz.list Fuzz.int) heavyAssertion + + +boolHeavyPass : Test +boolHeavyPass = + fuzz "(passes) bool, heavy assertion" Fuzz.bool <| + \b -> + heavyAssertion (List.range 0 60 |> List.filter (\n -> modBy 2 n == 0 || b)) + + +pairBoolHeavyPass : Test +pairBoolHeavyPass = + fuzz "(passes) pair of bools, heavy assertion" (Fuzz.pair Fuzz.bool Fuzz.bool) <| + \( a, b ) -> + heavyAssertion (List.range 0 60 |> List.filter (\n -> modBy 2 n == 0 || a || b)) + + +stringHeavyPass : Test +stringHeavyPass = + fuzz "(passes) string, heavy assertion" Fuzz.string <| + \s -> + heavyAssertion (String.toList s |> List.map Char.toCode) diff --git a/benchmarks/build.sh b/benchmarks/build.sh new file mode 100755 index 00000000..c273b118 --- /dev/null +++ b/benchmarks/build.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Compiles the throughput benchmarks. +# +# Copies the library from ../src into a local ELM_HOME so we benchmark *this* +# working copy rather than a published release. The old approach here -- putting +# ../src on source-directories -- cannot work: applications aren't allowed to +# import Elm.Kernel.* modules, so it stopped compiling as soon as RandomRun +# moved into the kernel. +# +# Elm caches compiled dependency artifacts per package *version*, and the +# version never changes as we edit ../src, so elm-stuff has to go too or the +# build silently reuses a stale library. + +set -euo pipefail + +cd "${0%/*}" + +PACKAGE_NAME=$(grep '"name"' ../elm.json | cut -d \" -f4) +PACKAGE_VERSION=$(grep '"version"' ../elm.json | cut -d \" -f4) +ELM_HOME_DIR="elm_home" +PACKAGE_PATH="${ELM_HOME_DIR}/0.19.2/packages/${PACKAGE_NAME}/${PACKAGE_VERSION}" + +rm -rf "${PACKAGE_PATH}" elm-stuff +mkdir -p "${PACKAGE_PATH}" +cp -r ../src "${PACKAGE_PATH}/src" +cp ../elm.json ../README.md ../LICENSE "${PACKAGE_PATH}/" + +ELM_HOME="${ELM_HOME_DIR}" elm make Main.elm --output elm.js + +echo "Built benchmarks/elm.js" diff --git a/benchmarks/elm.json b/benchmarks/elm.json index 1946fe98..6342bad5 100644 --- a/benchmarks/elm.json +++ b/benchmarks/elm.json @@ -1,28 +1,21 @@ { "type": "application", "source-directories": [ - ".", - "../src" + "." ], "elm-version": "0.19.2", "dependencies": { "direct": { - "elm/bytes": "1.0.8", "elm/core": "1.0.5", - "elm/html": "1.0.0", "elm/json": "1.1.3", "elm/random": "1.0.0", - "elm-explorations/benchmark": "1.0.2" + "elm-explorations/test": "2.2.1" }, "indirect": { - "BrianHicks/elm-trend": "2.1.3", - "elm/browser": "1.0.2", - "elm/regex": "1.0.0", + "elm/bytes": "1.0.8", + "elm/html": "1.0.0", "elm/time": "1.0.0", - "elm/url": "1.0.0", - "elm/virtual-dom": "1.0.2", - "mdgriffith/style-elements": "5.0.2", - "robinheghan/murmur3": "1.0.0" + "elm/virtual-dom": "1.0.2" } }, "test-dependencies": { diff --git a/benchmarks/runner.mjs b/benchmarks/runner.mjs new file mode 100755 index 00000000..d1034c16 --- /dev/null +++ b/benchmarks/runner.mjs @@ -0,0 +1,257 @@ +#!/usr/bin/env node + +// Drives the throughput benchmarks and reports runs/second. +// +// Usage: +// ./build.sh && node runner.mjs [options] +// +// --runs N fuzz runs per test (default 100, elm-test's own default). +// Try 1000 too: for a small-domain fuzzer the cost should +// grow with this, and shouldn't once exhaustive checking +// can stop early. +// --reps N timed repetitions per benchmark (default 5); the median +// is reported +// --min-ms N each repetition runs the test enough times to take at +// least this long, so the timer isn't the limiting factor +// (default 50) +// --filter REGEX only run benchmarks whose name matches +// --seed N fuzz seed (default 1) +// --label NAME recorded in the output +// --out FILE write the results as JSON +// --compare FILE diff against a saved result file +// --markdown emit markdown tables +// +// Reported as runs/sec (fuzz runs, not test invocations) so numbers stay +// comparable across different --runs settings. For a fuzzer whose domain is +// smaller than --runs, a future exhaustive implementation should show runs/sec +// going *up* with --runs, because the extra budget costs nothing. + +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; + +const here = path.dirname(fileURLToPath(import.meta.url)); + +function fail(message) { + console.error(`benchmarks: ${message}`); + process.exit(1); +} + +function parseArgs(argv) { + const options = { + runs: 100, + reps: 5, + minMs: 50, + filter: null, + seed: 1, + label: "current", + out: null, + compare: null, + markdown: false, + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const v = argv[++i]; + if (v === undefined) fail(`${arg} needs a value`); + return v; + }; + const nextInt = () => { + const v = Number(next()); + if (!Number.isInteger(v) || v < 0) fail(`${arg} needs a non-negative integer`); + return v; + }; + switch (arg) { + case "--runs": options.runs = nextInt(); break; + case "--reps": options.reps = nextInt(); break; + case "--min-ms": options.minMs = nextInt(); break; + case "--seed": options.seed = nextInt(); break; + case "--filter": options.filter = new RegExp(next()); break; + case "--label": options.label = next(); break; + case "--out": options.out = next(); break; + case "--compare": options.compare = next(); break; + case "--markdown": options.markdown = true; break; + default: fail(`unknown option: ${arg}`); + } + } + return options; +} + +function startElm() { + const require = createRequire(import.meta.url); + const elmPath = path.join(here, "elm.js"); + if (!fs.existsSync(elmPath)) fail("elm.js not found -- run ./build.sh first"); + const { Elm } = require(elmPath); + const app = Elm.Main.init(); + + let pending = null; + app.ports.toJs.subscribe((message) => { + const resolve = pending; + pending = null; + if (resolve === null) fail(`unexpected message: ${JSON.stringify(message)}`); + resolve(message); + }); + + return (request) => + new Promise((resolve) => { + if (pending !== null) fail("request sent while another was in flight"); + pending = resolve; + app.ports.fromJs.send(request); + }); +} + +// One timed repetition: `iterations` whole test executions, returning ms elapsed. +// The Elm work happens synchronously inside the port handler, so the round trip +// is the work plus a microtask -- negligible once a repetition takes >= minMs. +async function timeOnce(send, index, options, iterations) { + const started = performance.now(); + const result = await send({ tag: "run", index, runs: options.runs, seed: options.seed, iterations }); + const elapsed = performance.now() - started; + if (result.tag !== "done") fail(`expected done, got ${JSON.stringify(result)}`); + return elapsed; +} + +async function measure(options) { + const send = startElm(); + const manifest = await send({ tag: "manifest" }); + if (manifest.tag !== "manifest") fail(`expected a manifest, got ${JSON.stringify(manifest)}`); + + const selected = manifest.benchmarks + .map((b, index) => ({ ...b, index })) + .filter(({ name }) => options.filter === null || options.filter.test(name)); + if (selected.length === 0) fail("no benchmarks to run"); + + const results = []; + const startedAt = Date.now(); + for (const { name, passes, index } of selected) { + // Grow the iteration count until a repetition clears --min-ms, so the timer + // resolution isn't what we're measuring. Also serves as warmup. + let iterations = 1; + while ((await timeOnce(send, index, options, iterations)) < options.minMs) { + iterations *= 2; + if (iterations > 1 << 22) break; + } + + const samples = []; + for (let r = 0; r < options.reps; r++) { + samples.push(await timeOnce(send, index, options, iterations)); + } + samples.sort((a, b) => a - b); + const medianMs = samples[Math.floor(samples.length / 2)]; + + results.push({ + name, + passes, + iterations, + medianMs, + msPerTest: medianMs / iterations, + // Fuzz runs per second, comparable across --runs settings. Null for a + // failing test, which never executes its whole budget. + runsPerSec: passes ? (iterations * options.runs) / (medianMs / 1000) : null, + spreadPct: ((samples[samples.length - 1] - samples[0]) / medianMs) * 100, + }); + } + + return { + meta: { + label: options.label, + runs: options.runs, + reps: options.reps, + minMs: options.minMs, + seed: options.seed, + node: process.version, + startedAt: new Date(startedAt).toISOString(), + wallMs: Date.now() - startedAt, + }, + results, + }; +} + +let MARKDOWN = false; + +function table(rows, headers, aligns) { + if (MARKDOWN) { + const esc = (c) => String(c).replace(/\|/g, "\\|"); + const row = (cells) => `| ${cells.map(esc).join(" | ")} |`; + return [row(headers), `| ${aligns.map((a) => (a === "r" ? "---:" : ":---")).join(" | ")} |`, ...rows.map(row)].join("\n"); + } + const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => String(r[i]).length))); + const line = (cells) => + cells.map((c, i) => (aligns[i] === "r" ? String(c).padStart(widths[i]) : String(c).padEnd(widths[i]))).join(" "); + return [line(headers), line(widths.map((w) => "-".repeat(w))), ...rows.map(line)].join("\n"); +} + +function si(n) { + if (n >= 1e6) return `${(n / 1e6).toFixed(2)}M`; + if (n >= 1e3) return `${(n / 1e3).toFixed(1)}k`; + return n.toFixed(0); +} + +function report(run) { + const lines = []; + lines.push( + MARKDOWN + ? `## Throughput: ${run.meta.label}\n\n--runs ${run.meta.runs}, ${run.meta.reps} reps, ${(run.meta.wallMs / 1000).toFixed(1)}s wall.` + : `Throughput: ${run.meta.label} -- --runs ${run.meta.runs}, ${run.meta.reps} reps, ${(run.meta.wallMs / 1000).toFixed(1)}s wall`, + ); + lines.push(""); + lines.push( + table( + run.results.map((r) => [ + r.name, + r.runsPerSec === null ? "n/a" : si(r.runsPerSec), + r.msPerTest.toFixed(4), + String(r.iterations), + `${r.spreadPct.toFixed(0)}%`, + ]), + ["benchmark", "runs/sec", "ms/test", "iters", "spread"], + ["l", "r", "r", "r", "r"], + ), + ); + return lines.join("\n"); +} + +function compareReport(current, baseline) { + const byName = new Map(baseline.results.map((r) => [r.name, r])); + const rows = []; + const ratios = []; + for (const r of current.results) { + const b = byName.get(r.name); + if (b === undefined) { + rows.push([r.name, "new", r.msPerTest.toFixed(4), "-"]); + continue; + } + // Compare on ms/test, which is defined for passing and failing alike. + const ratio = b.msPerTest / r.msPerTest; + ratios.push(ratio); + rows.push([r.name, b.msPerTest.toFixed(4), r.msPerTest.toFixed(4), `${ratio.toFixed(2)}x`]); + } + const geo = ratios.length === 0 ? null : Math.exp(ratios.reduce((s, v) => s + Math.log(v), 0) / ratios.length); + const lines = []; + lines.push(MARKDOWN ? `## Throughput: ${current.meta.label} vs ${baseline.meta.label}` : `Throughput: ${current.meta.label} vs ${baseline.meta.label}`); + if (current.meta.runs !== baseline.meta.runs) { + lines.push(`WARNING: --runs differs (${current.meta.runs} vs ${baseline.meta.runs}); runs/sec is comparable but ms/test is not.`); + } + lines.push(""); + lines.push(table(rows, ["benchmark", "ms/test (base)", "ms/test", "speedup"], ["l", "r", "r", "r"])); + lines.push(""); + lines.push(`Geometric mean throughput ratio: ${geo === null ? "-" : geo.toFixed(3) + "x"} (> 1 is faster)`); + return lines.join("\n"); +} + +const options = parseArgs(process.argv.slice(2)); +MARKDOWN = options.markdown; + +const run = await measure(options); +console.log(report(run)); +if (options.out !== null) { + fs.mkdirSync(path.dirname(options.out), { recursive: true }); + fs.writeFileSync(options.out, JSON.stringify(run, null, 2) + "\n"); + console.log(`\nWrote ${path.relative(process.cwd(), options.out)}`); +} +if (options.compare !== null) { + console.log(""); + console.log(compareReport(run, JSON.parse(fs.readFileSync(options.compare, "utf8")))); +} diff --git a/elm.json b/elm.json index 68b845ca..a46cfcbc 100644 --- a/elm.json +++ b/elm.json @@ -7,6 +7,7 @@ "exposed-modules": [ "Test", "Test.Runner", + "Test.RunnerV2", "Test.Runner.Failure", "Test.Distribution", "Expect", diff --git a/f-metric/.gitignore b/f-metric/.gitignore new file mode 100644 index 00000000..baa5287d --- /dev/null +++ b/f-metric/.gitignore @@ -0,0 +1,4 @@ +elm-stuff/ +elm_home/ +elm.js +results/ diff --git a/f-metric/README.md b/f-metric/README.md new file mode 100644 index 00000000..86f5da7a --- /dev/null +++ b/f-metric/README.md @@ -0,0 +1,302 @@ +# F-metric harness + +Measures how good elm-test is at **finding bugs**, as opposed to how fast it runs +(that's `benchmarks/`). + +The corpus is a set of fuzz tests with deliberately injected defects. For each +one, across many seeds, we record: + +| metric | why | +| --- | --- | +| **time to a minimal counterexample** | the headline. What the developer actually waits for: detection _and_ simplification. | +| **cases to the defect** | the F-metric in its original sense: how many values the fuzzer generated before the property broke. Exact integers, and fully deterministic given the fixed seed list, so this is the axis with no measurement noise at all. | +| **detection rate at the budget** | whether the defect was found at all within `runs`. A strategy that finds a bug in 3 cases instead of 10 000 has won nothing if it's slower in wall-clock terms — but a strategy that stops finding the bug at all has lost something real. Only `cache/truncated-key` is tuned to sit below 100%, which is what makes this column informative. | +| **counterexample quality** | whether the value we report is the best possible one, plus the length and sum of the underlying `RandomRun` (the two components of the shortlex order that simplifying minimises). | + +Time is the headline because it's what a developer actually experiences, and the +corpus shows why the two axes can't substitute for each other: +`needle/int-in-range` takes 370 cases but 0.12 ms, while `challenge/deletion` +takes 4 cases but 9 ms. Cases measure how well generation searches; time measures +what that search costs. Read them together — and prefer cases when they +disagree, because the case count is exact and the timings are not. + +## Running + +```sh +cd f-metric +./build.sh +node runner.mjs --seeds 100 +``` + +`./build.sh` copies `../src` into a local `ELM_HOME` and compiles against it, so +you're always measuring the working copy. (The trick `benchmarks/` uses — putting +`../src` on `source-directories` — cannot work, because applications aren't +allowed to import `Elm.Kernel.*` modules. That's also why `benchmarks/` currently +doesn't compile.) + +A 300-seed run over the default corpus takes about 30 seconds. `npm run f-metric` +from the repo root does the build and run in one step. Adding `--shrink-only` +makes it minutes rather than seconds, because `challenge/length-list` spends over +a second per measurement simplifying a 100-element list. + +### Comparing two implementations + +```sh +# on master +./build.sh && node runner.mjs --seeds 100 --label before --save-summary /tmp/before.json + +# on your branch +./build.sh && node runner.mjs --seeds 100 --label after --compare /tmp/before.json +``` + +The comparison prints per-case detection-rate and time deltas, plus a geometric +mean of the time ratios (geometric, so that one slow case can't dominate). + +Add `--markdown` to any report or comparison to get GitHub-flavoured tables for +pasting into a PR or issue. + +`baseline/` holds committed summaries (the small per-case aggregates, not the raw +measurements) so you can diff against a known state without rerunning the other +side. `pr260-random.json` is today's purely-random generation, 300 seeds, budget +10 000, measured on an M-series Mac; `pr260-random-with-shrink.json` is the same +with `--shrink-only` included. **Absolute times depend on the machine**, so +for a real before/after, measure both sides yourself on one machine and use the +committed baseline only as a sanity check on the shape of the results — mainly +`found` and `optimal`, which are machine-independent. + +## Reading the output + +``` +case found med cases p75 med ms p75 ms optimal |run| sum(run) +needle/int-in-range 100% 370 628 0.13 0.21 100% 2 43 +``` + +- `found` — fraction of seeds that detected the defect within the budget. +- `med cases` / `p75` — values generated before the property broke. **The + noise-free axis**: for a fixed seed list this is deterministic, so a change + here is entirely attributable to the change under test, with no need to argue + about measurement error. +- `med ms` / `p75 ms` — time to a minimal counterexample, **among the seeds that + detected it**. A censored seed spent the entire budget and produced no + counterexample; averaging that in as if it were a time-to-failure would be + meaningless, so it isn't. Watch `found` and `med ms` together: a change that + halves the time while dropping `found` from 100% to 60% is a regression. +- `optimal` — fraction of detections that reported the best possible + counterexample. `-` means we haven't written that case's optimum down. +- `|run|`, `sum(run)` — median length and sum of the reported `RandomRun`. Lower + is a simpler counterexample. These work even for cases with no known optimum, + and they're directly comparable across strategies because they're the very + quantities simplifying tries to minimise. + +A case with `med cases` of 1 or 2 was falsified by the first value or two the +fuzzer produced, so it measures *only* simplification cost, not search quality. +Those cases are tagged `shrinkOnly` in the corpus and **excluded by default**; +`--shrink-only` puts them back. Removing them took the default run from 204 +seconds to about 10 (at 100 seeds), almost all of it `challenge/length-list`. + +## Methodology, and how much to trust a number + +**The noise floor is measured, not guessed**, and the two axes are in completely +different regimes. + +The **case count carries no noise at all**. Re-running an identical +configuration gives a case ratio of exactly 1.000x on every single case, at 100 +and at 500 seeds — the seed list is fixed and generation is deterministic. Any +movement in `cases` is real, with no error bars to argue about. This is the axis +to compare on. + +The **detection rate carries no run-to-run noise either**, for the same reason: +re-running gives byte-identical `found` values. But read it with one caveat the +case ratio doesn't need — it's a proportion over 300 seeds, so its sampling error +as an estimate of the *true* detection rate is about ±3 percentage points +(1 s.e.). A 2-3 point difference between two implementations is real on this seed +list but may not generalise; a 10 point difference is a result. + +The **times cannot be compared across process invocations to better than about +±7%**, and more seeds do not fix it. Two back-to-back 500-seed runs of the +identical configuration came out uniformly 0.86x–1.03x, clustered around 0.93x: +not per-case scatter but a systematic whole-run drift, the second process simply +running faster than the first. Within a run the times are fine; between runs +they carry a bias you can't average away. Treat a time change under about 1.15x +per case, or 1.1x in aggregate, as nothing. + +Cases whose median is under 0.5 ms are **excluded from the aggregate time ratio** +and flagged with `*`, because a single `performance.now()` tick is a double-digit +percentage of them — a 0.02 ms case reading 1.29x is not information. With the +shrink-only cases excluded that removes 11 of 18 cases from the time aggregate, +which leaves it resting on few enough cases to be fragile. Two consequences: +compare on `cases`, and if you specifically care about *simplification* time, run +with `--shrink-only`, since the slow cases all live there. + +Things the harness does to keep comparisons honest, and which you shouldn't +undo: + +- **Fixed seeds.** Seeds come from a fixed xorshift32 sequence, so every run + measures the same seeds. Consecutive integers would be a poor choice — + `Random.initialSeed` does little mixing, so nearby seeds can correlate. +- **Warmup.** The first few measurements of each case are discarded + (`--warmup`), so a case isn't penalised for being the one that triggered JIT + compilation. +- **Seed-major ordering.** Cases are interleaved rather than run to completion + one at a time, so drift over a long run (thermal throttling, GC pressure) is + spread across all cases instead of landing on whichever case ran last. +- **Same everything on both sides.** `--filter`, `--seeds`, `--warmup` and the + budget all affect measured times — a filtered run measures each case as much as + 2.5x slower than the same case in a full run, purely because of how much other + code the JIT saw. The comparison warns about mismatched budgets and seed + counts, but it can't detect a mismatched `--filter`. +- **Dev mode, which is also production.** We don't compile with `--optimize`, + and there's no option to. That isn't a readability compromise — it's what + node-test-runner does: `lib/ElmCompiler.js` only ever passes `--output` and + `--report json`. It has to, because under `--optimize` `Debug.toString` + returns `""` for _every_ value, plain `Int` included, so every + counterexample elm-test reports would be unreadable. Dev mode is the + configuration real test suites run in, so these timings are representative of + it. + +Known quirks of the current corpus: + +- `cache/truncated-key` sits at **55% found** by design; everything else is at + 100%. Without it the `found` column would be a constant and tell you nothing. + Its mask is tuned so a run collides with probability ~`28 / 2^19`, which over a + 10 000-run budget lands about half the seeds either side — so the axis responds + in both directions. One mask bit is worth roughly a factor of two in `found` + (18 bits measured 80%, 19 bits 55%), so retune the mask, never this case's + budget, if it drifts to an extreme. +- Everything else detects on every seed at the default budget. That's deliberate + headroom: `found` starts at 100% and has somewhere to fall if a change makes + generation worse. +- `bst/delete-two-children` reports **1% optimal**. That's not a broken + expectation — simplifying reliably finds the right four-node *shape* but never + reduces the keys, stopping at `([5,0,7,6],5)` instead of `([1,0,3,2],1)`. It's + the most sensitive simplification signal in the default corpus, and the 1% + proves the target is reachable. +- With `--shrink-only`, `challenge/coupling` (53% optimal) and + `structural/adjacent-dedupe` (32%) are the other two places simplifying + reliably stops short. + +## The corpus + +Every case is a fuzz test with one deliberately injected defect. Cases are tagged +with a **role**: `Search` cases need the fuzzer to work to find the defect, so +their case counts and times mean something; `ShrinkOnly` cases are falsified by +the first value or two and only measure simplification, so they're excluded +unless you pass `--shrink-only`. + +`src/Corpus/KnownBugs.elm` — data structures with one realistic mistake, and the +cases that carry most of the F signal. Each needs a *sequence* of operations that +interact, so no single value falsifies them: + +| case | the defect | source | +| --- | --- | --- | +| `bst/delete-two-children` | deleting a two-child node replaces it with the right *child's* key instead of the right *subtree's minimum* — the same thing exactly when that child has no left subtree | the BST suite in John Hughes, [_How to Specify It!_](https://research.chalmers.se/publication/508940) (2019) | +| `rbt/missing-rotation` | Okasaki's `balance` with the left-right case removed; the other three still rebalance most insertions | Okasaki's red-black tree with the `balance` mutation set from [Etna](https://dl.acm.org/doi/10.1145/3591283) (Shi et al., PLDI 2023) | +| `intervals/touching-merge` | interval merge tests `lo < previousHi` where it needs `<=`, so intervals meeting at exactly one point aren't merged | classic bug class, not one named incident | +| `queue/missing-invariant` | Okasaki's batched queue where `pop` forgets to restore "front is empty only if the queue is", so a non-empty queue reports itself empty | classic bug class; the textbook motivation for model-based testing | +| `cache/truncated-key` | a cache keys entries on the low 19 bits of a 24-bit id, so two distinct ids collide and one silently overwrites the other | classic bug class. Also the corpus's **only case that isn't always found** — see below | + +Two of these needed their input length bounded (`listOfLengthBetween 0 6`) to be +worth measuring. With `Fuzz.list`'s ~16 elements, a random 16-key sequence almost +always contains the trigger somewhere and the case is falsified by the very first +value — bounding the size makes *finding* the trigger the work, which is the +point, and it's how Etna's suites bound their inputs too. + +`src/Corpus/Combined.elm` — cases where part of the domain is tiny and part is +huge, with the defect in the huge part. This is +[#188](https://github.com/elm-explorations/test/issues/188)'s motivating shape: +given `oneOf [map Err string, map Ok bool]`, half of every run is spent +re-testing `Ok True` and `Ok False`, so a defect in the `Err` branch is found at +half the rate it could be. + +The three cases vary exactly one thing each, so a measurement says *which* +mechanism paid off rather than just that something did: + +| case | `Err` payload | choice tree | helped by | +| :--- | :--- | :--- | :--- | +| `combined/open-payload` | `Fuzz.string` | infinite | duplicate rejection, or a per-subtree hybrid. **Not** all-or-nothing enumeration, which abandons an infinite tree and changes nothing. | +| `combined/finite-payload` | uniform `0..1023` | finite, 1026 leaves | plain exhaustive checking: guaranteed detection with zero variance | +| `combined/wide-easy-branch` | same, but `Ok` has 201 leaves | finite, 1225 leaves | separates leaf-level dedup from subtree-level exhaustion — dedup must *observe* all 201 `Ok` runs first (~`201*H(201)` ≈ 1200 draws) where an enumerator knows when the branch is covered | + +`finite-payload` and `wide-easy-branch` have identical baselines by construction +(`Fuzz.bool` and `intRange 0 200` both consume exactly one draw, so the `Err` +sequence is bit-identical). They only diverge once a mechanism lands, which makes +them a matched pair. + +`src/Corpus/Synthetic.elm` — cases shaped to probe one weakness each: + +| category | what it rewards | +| --- | --- | +| `needle` | systematically covering a small domain | +| `boundary` | trying range edges and size edges deliberately | +| `structural` | producing a required *shape* (a repeat, a nesting) rather than a value | +| `deep` | reaching *large* inputs. **The category to watch** — anything that biases generation towards small inputs will look wonderful everywhere else and regress here. | +| `filter` | the rejection path in the fuzz loop | + +`src/Corpus/ShrinkingChallenge.elm` — the +[shrinking-challenge](https://github.com/jlink/shrinking-challenge) suite, ported +from `tests/src/ShrinkingChallengeTests.elm`, which already encodes each +challenge as (fuzzer, falsifiable property, known-optimal counterexample). Keep +the two in sync. Seven of the twelve are `shrinkOnly`: they were designed to +stress *simplifying*, not detection, so the first generated value falsifies them. +The five that survive as search cases are `calculator`, `deletion` and the three +`difference` variants. + +### Candidates not taken + +Worth recording so nobody re-derives the dead ends: + +- **Case-folding bugs** (the Turkish dotless `i`, German `ß` changing length + under `toUpper`) are a famous, security-adjacent class, but unreachable here: + `Fuzz.char` only draws from the full Unicode range 1/10 of the time, so a + specific code point is about 1e-7 per character. The case would be censored + forever and discriminate nothing. +- **The TimSort `mergeCollapse` bug** (de Gouw et al. 2015, which broke Java and + Python) needs a specific run-length pattern across 60+ elements. Faithful + replication is a lot of code for an input no random fuzzer will reach. + +### Adding a case + +See the docs on `Corpus.Case.fails`. Briefly: the property must be falsifiable, +the defect should be reachable but not trivially so, and `minima` should list the +counterexamples you'd consider optimal (as `Debug.toString` renders them). Leave +`minima` empty, run once, and read the `given` values out of the report to fill +it in. + +Two traps, both of which this corpus hit: + +- **Make sure the defect is reachable at all.** `Fuzz.string` is + `stringOfLengthBetween 0 10`, so a property that only fails for strings longer + than 10 characters is never falsifiable — it looks like a hard case but it's an + impossible one. The report calls out cases that were never detected. +- **Keep `Fuzz.filter` predicates generous.** `Fuzz.filter` fails the whole test + after 16 consecutive rejections, and over thousands of runs even a mildly + selective predicate will hit that. A predicate rejecting 76% of values fails + spuriously within a couple of thousand runs. Such a failure is _not_ a + detection, and the harness reports it separately as "failed for the wrong + reason" — that's what the `F-METRIC-VIOLATION` marker is for. + +## Where the case count comes from + +Reporting cases needs one number out of the library that wasn't previously +reachable: `Test/Fuzz.elm` tracks `runsElapsed` in its loop state, but it only +ever escaped to a runner inside a `DistributionReport`, i.e. only for tests that +asked for a distribution. This branch threads it through `RunResult` and +`Test.Expectation.FuzzTestFail` and exposes it as +`Test.RunnerV2.getFuzzTestFailRunsElapsed`. + +That is a change to `src/`, and an addition to PR #260's public API, so it has to +land with or after #260. It's deliberately minimal — one field, one getter, no +behaviour change — and the existing 727-test suite passes unchanged. + +Two things worth knowing about the number: + +- It counts values *generated*, which includes ones the fuzzer rejected, so for + cases using `Fuzz.filter` it's larger than the number of values the property + actually saw. That's the right denominator for "how much work did the test + system do". +- It can exceed the configured `runs`, because an `expectDistribution` test keeps + generating until its statistical check settles. + +An obvious follow-up, not done here: expose the same count for *passing* tests. +It's free now that `RunResult` carries it, and it would let a runner say "this +test actually ran 3072 times" for distribution-heavy tests. diff --git a/f-metric/baseline/pr260-random-with-shrink.json b/f-metric/baseline/pr260-random-with-shrink.json new file mode 100644 index 00000000..e39520c9 --- /dev/null +++ b/f-metric/baseline/pr260-random-with-shrink.json @@ -0,0 +1,607 @@ +{ + "tag": "summary", + "meta": { + "label": "baseline-shrink", + "startedAt": "2026-09-29T16:28:29.750Z", + "wallMs": 678922, + "seeds": 300, + "warmup": 3, + "budget": 10000, + "shrinkOnly": true, + "node": "v22.22.2", + "cases": [ + "bst/delete-two-children", + "rbt/missing-rotation", + "intervals/touching-merge", + "queue/missing-invariant", + "cache/truncated-key", + "combined/open-payload", + "combined/finite-payload", + "combined/wide-easy-branch", + "needle/int-in-range", + "needle/pair-diagonal", + "needle/exact-list", + "boundary/range-top", + "boundary/last-chunk", + "structural/adjacent-dedupe", + "structural/just-nothing", + "deep/list-sum", + "deep/big-int", + "deep/long-string", + "filter/multiple-of-four", + "challenge/reverse", + "challenge/large-union-list", + "challenge/calculator", + "challenge/length-list", + "challenge/difference1", + "challenge/difference2", + "challenge/difference3", + "challenge/bin-heap", + "challenge/coupling", + "challenge/deletion", + "challenge/distinct", + "challenge/nested-lists" + ], + "brokenCases": [] + }, + "cases": [ + { + "name": "boundary/last-chunk", + "category": "boundary", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.10829149999699439, + "p25Ms": 0.05500000005122274, + "p75Ms": 0.330989749996661, + "medianCases": 1, + "p75Cases": 2, + "medianRunLength": 3, + "medianRunSum": 1, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "boundary/range-top", + "category": "boundary", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.019896000012522563, + "p25Ms": 0.01275000003806781, + "p75Ms": 0.03434349998860853, + "medianCases": 65.5, + "p75Cases": 142, + "medianRunLength": 1, + "medianRunSum": 100, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "bst/delete-two-children", + "category": "known-bug", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 1.4057080000056885, + "p25Ms": 1.1893644999872777, + "p75Ms": 1.6764164999767672, + "medianCases": 19, + "p75Cases": 41, + "medianRunLength": 10, + "medianRunSum": 75, + "optimalRate": 0.013333333333333334, + "censoredMs": null + }, + { + "name": "cache/truncated-key", + "category": "known-bug", + "n": 300, + "detected": 164, + "errors": 0, + "errorDescription": null, + "detectionRate": 0.5466666666666666, + "medianMs": 32.91060400000424, + "p25Ms": 16.33945774997119, + "p75Ms": 54.24508350001997, + "medianCases": 3593, + "p75Cases": 6393, + "medianRunLength": 17, + "medianRunSum": 76, + "optimalRate": null, + "censoredMs": 79.80529149999347 + }, + { + "name": "challenge/bin-heap", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.8875835000071675, + "p25Ms": 0.668082999989565, + "p75Ms": 1.3041980000125477, + "medianCases": 1, + "p75Cases": 1, + "medianRunLength": 8, + "medianRunSum": 3, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/calculator", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 2.145749999966938, + "p25Ms": 1.504322249937104, + "p75Ms": 3.0741349999880185, + "medianCases": 3, + "p75Cases": 5, + "medianRunLength": 11, + "medianRunSum": 3, + "optimalRate": 0.9833333333333333, + "censoredMs": null + }, + { + "name": "challenge/coupling", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 1.630916000023717, + "p25Ms": 1.135999999998603, + "p75Ms": 2.539186999980302, + "medianCases": 2, + "p75Cases": 4, + "medianRunLength": 5, + "medianRunSum": 3, + "optimalRate": 0.5733333333333334, + "censoredMs": null + }, + { + "name": "challenge/deletion", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 9.74204149999423, + "p25Ms": 4.525291500031017, + "p75Ms": 15.789780750001228, + "medianCases": 4, + "p75Cases": 7, + "medianRunLength": 8, + "medianRunSum": 2, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/difference1", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.2845209999995859, + "p25Ms": 0.16507274998002686, + "p75Ms": 0.7209379999840166, + "medianCases": 220.5, + "p75Cases": 452.75, + "medianRunLength": 4, + "medianRunSum": 20, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/difference2", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.10962449997896329, + "p25Ms": 0.08446900006674696, + "p75Ms": 0.4373539999942295, + "medianCases": 31, + "p75Cases": 60, + "medianRunLength": 4, + "medianRunSum": 16, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/difference3", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.2117294999989099, + "p25Ms": 0.11541699998997501, + "p75Ms": 0.5862599999309168, + "medianCases": 124, + "p75Cases": 252.25, + "medianRunLength": 4, + "medianRunSum": 19, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/distinct", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.6823335000080988, + "p25Ms": 0.5415517499932321, + "p75Ms": 1.0057497500238242, + "medianCases": 1, + "p75Cases": 1, + "medianRunLength": 10, + "medianRunSum": 8, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/large-union-list", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 22.54014600002847, + "p25Ms": 8.13009374999092, + "p75Ms": 72.0059792500142, + "medianCases": 1, + "p75Cases": 1, + "medianRunLength": 18, + "medianRunSum": 20, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/length-list", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 1813.5789580000273, + "p25Ms": 599.1133849999605, + "p75Ms": 3250.0773752500045, + "medianCases": 2, + "p75Cases": 3, + "medianRunLength": 5, + "medianRunSum": 6909, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/nested-lists", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 10.321812000009231, + "p25Ms": 6.771999999997206, + "p75Ms": 19.08112525001343, + "medianCases": 1, + "p75Cases": 1, + "medianRunLength": 36, + "medianRunSum": 12, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/reverse", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.3842084999905637, + "p25Ms": 0.2614994999967166, + "p75Ms": 0.6669687499961583, + "medianCases": 1, + "p75Cases": 1, + "medianRunLength": 7, + "medianRunSum": 4, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "combined/finite-payload", + "category": "combined", + "n": 300, + "detected": 299, + "errors": 0, + "errorDescription": null, + "detectionRate": 0.9966666666666667, + "medianMs": 0.4442080000007991, + "p25Ms": 0.17650000000503496, + "p75Ms": 0.9336665000300854, + "medianCases": 1307, + "p75Cases": 2638, + "medianRunLength": 3, + "medianRunSum": 38, + "optimalRate": null, + "censoredMs": 3.157915999996476 + }, + { + "name": "combined/open-payload", + "category": "combined", + "n": 300, + "detected": 286, + "errors": 0, + "errorDescription": null, + "detectionRate": 0.9533333333333334, + "medianMs": 5.609395499996026, + "p25Ms": 3.0134690000049886, + "p75Ms": 10.082718749996275, + "medianCases": 2358.5, + "p75Cases": 4392.5, + "medianRunLength": 8, + "medianRunSum": 173, + "optimalRate": null, + "censoredMs": 18.838458499958506 + }, + { + "name": "combined/wide-easy-branch", + "category": "combined", + "n": 300, + "detected": 299, + "errors": 0, + "errorDescription": null, + "detectionRate": 0.9966666666666667, + "medianMs": 0.4612080000224523, + "p25Ms": 0.18147899996620254, + "p75Ms": 0.9178960000026564, + "medianCases": 1307, + "p75Cases": 2638, + "medianRunLength": 3, + "medianRunSum": 38, + "optimalRate": null, + "censoredMs": 3.350458000000799 + }, + { + "name": "deep/big-int", + "category": "deep", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.23629149999760557, + "p25Ms": 0.1958539999977802, + "p75Ms": 0.28197924996493384, + "medianCases": 22, + "p75Cases": 44, + "medianRunLength": 2, + "medianRunSum": 2000015, + "optimalRate": null, + "censoredMs": null + }, + { + "name": "deep/list-sum", + "category": "deep", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 24.402458500000648, + "p25Ms": 17.563333749993035, + "p75Ms": 33.00617700000021, + "medianCases": 3, + "p75Cases": 5, + "medianRunLength": 25, + "medianRunSum": 112, + "optimalRate": null, + "censoredMs": null + }, + { + "name": "deep/long-string", + "category": "deep", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 48.79222900001332, + "p25Ms": 39.60070824998547, + "p75Ms": 59.523406249998516, + "medianCases": 3, + "p75Cases": 6, + "medianRunLength": 64, + "medianRunSum": 33, + "optimalRate": null, + "censoredMs": null + }, + { + "name": "filter/multiple-of-four", + "category": "filter", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.045479000018531224, + "p25Ms": 0.03494774999217043, + "p75Ms": 0.060916749978787266, + "medianCases": 52, + "p75Cases": 98.5, + "medianRunLength": 1, + "medianRunSum": 42, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "intervals/touching-merge", + "category": "known-bug", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 1.0286869999981718, + "p25Ms": 0.71524999999383, + "p75Ms": 1.5888642499994603, + "medianCases": 11, + "p75Cases": 22, + "medianRunLength": 7, + "medianRunSum": 2, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "needle/exact-list", + "category": "needle", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.4541869999957271, + "p25Ms": 0.25803100004122825, + "p75Ms": 0.741885249997722, + "medianCases": 481, + "p75Cases": 914.5, + "medianRunLength": 7, + "medianRunSum": 9, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "needle/int-in-range", + "category": "needle", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.10256249999292777, + "p25Ms": 0.049750250022043474, + "p75Ms": 0.1695097499905387, + "medianCases": 330, + "p75Cases": 561, + "medianRunLength": 2, + "medianRunSum": 43, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "needle/pair-diagonal", + "category": "needle", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.02183300000615418, + "p25Ms": 0.015635750081855804, + "p75Ms": 0.032062750004115514, + "medianCases": 33, + "p75Cases": 66.5, + "medianRunLength": 2, + "medianRunSum": 0, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "queue/missing-invariant", + "category": "known-bug", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.6338749999995343, + "p25Ms": 0.5776039999909699, + "p75Ms": 0.7265312500094296, + "medianCases": 17, + "p75Cases": 32.25, + "medianRunLength": 11, + "medianRunSum": 6, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "rbt/missing-rotation", + "category": "known-bug", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.30118800001218915, + "p25Ms": 0.278406499999619, + "p75Ms": 0.34044724999694154, + "medianCases": 8, + "p75Cases": 16, + "medianRunLength": 7, + "medianRunSum": 6, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "structural/adjacent-dedupe", + "category": "structural", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.3761044999992009, + "p25Ms": 0.29159350000554696, + "p75Ms": 0.5079897500254447, + "medianCases": 1, + "p75Cases": 2, + "medianRunLength": 7, + "medianRunSum": 5, + "optimalRate": 0.31, + "censoredMs": null + }, + { + "name": "structural/just-nothing", + "category": "structural", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.013792000005196314, + "p25Ms": 0.011792000026616734, + "p75Ms": 0.016593499967711978, + "medianCases": 4, + "p75Cases": 7, + "medianRunLength": 2, + "medianRunSum": 1, + "optimalRate": 1, + "censoredMs": null + } + ] +} diff --git a/f-metric/baseline/pr260-random.json b/f-metric/baseline/pr260-random.json new file mode 100644 index 00000000..95c3c737 --- /dev/null +++ b/f-metric/baseline/pr260-random.json @@ -0,0 +1,436 @@ +{ + "tag": "summary", + "meta": { + "label": "baseline", + "startedAt": "2026-09-29T15:24:30.032Z", + "wallMs": 46372, + "seeds": 300, + "warmup": 3, + "budget": 10000, + "shrinkOnly": false, + "node": "v22.22.2", + "cases": [ + "bst/delete-two-children", + "rbt/missing-rotation", + "intervals/touching-merge", + "queue/missing-invariant", + "cache/truncated-key", + "combined/open-payload", + "combined/finite-payload", + "combined/wide-easy-branch", + "needle/int-in-range", + "needle/pair-diagonal", + "needle/exact-list", + "boundary/range-top", + "structural/just-nothing", + "deep/list-sum", + "deep/big-int", + "deep/long-string", + "filter/multiple-of-four", + "challenge/calculator", + "challenge/difference1", + "challenge/difference2", + "challenge/difference3", + "challenge/deletion" + ], + "brokenCases": [] + }, + "cases": [ + { + "name": "boundary/range-top", + "category": "boundary", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.018667000000050393, + "p25Ms": 0.012020249998386134, + "p75Ms": 0.032531249998100975, + "medianCases": 65.5, + "p75Cases": 142, + "medianRunLength": 1, + "medianRunSum": 100, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "bst/delete-two-children", + "category": "known-bug", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 1.2941249999967113, + "p25Ms": 1.1184379999995144, + "p75Ms": 1.5594167499994, + "medianCases": 19, + "p75Cases": 41, + "medianRunLength": 10, + "medianRunSum": 75, + "optimalRate": 0.013333333333333334, + "censoredMs": null + }, + { + "name": "cache/truncated-key", + "category": "known-bug", + "n": 300, + "detected": 164, + "errors": 0, + "errorDescription": null, + "detectionRate": 0.5466666666666666, + "medianMs": 30.17560400000002, + "p25Ms": 14.107968499999515, + "p75Ms": 50.42684350000127, + "medianCases": 3593, + "p75Cases": 6393, + "medianRunLength": 17, + "medianRunSum": 76, + "optimalRate": null, + "censoredMs": 71.06839599999967 + }, + { + "name": "challenge/calculator", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 2.1331255000004603, + "p25Ms": 1.4414064999995162, + "p75Ms": 2.9805102499976783, + "medianCases": 3, + "p75Cases": 5, + "medianRunLength": 11, + "medianRunSum": 3, + "optimalRate": 0.9833333333333333, + "censoredMs": null + }, + { + "name": "challenge/deletion", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 8.972979500002111, + "p25Ms": 4.574343499999827, + "p75Ms": 15.16271924999728, + "medianCases": 4, + "p75Cases": 7, + "medianRunLength": 8, + "medianRunSum": 2, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/difference1", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.25743749999855936, + "p25Ms": 0.14927075000105106, + "p75Ms": 0.6639065000008486, + "medianCases": 220.5, + "p75Cases": 452.75, + "medianRunLength": 4, + "medianRunSum": 20, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/difference2", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.10377099999914208, + "p25Ms": 0.078989749998982, + "p75Ms": 0.4573232500008544, + "medianCases": 31, + "p75Cases": 60, + "medianRunLength": 4, + "medianRunSum": 16, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "challenge/difference3", + "category": "challenge", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.19149999999899592, + "p25Ms": 0.11370825000130935, + "p75Ms": 0.6018850000000384, + "medianCases": 124, + "p75Cases": 252.25, + "medianRunLength": 4, + "medianRunSum": 19, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "combined/finite-payload", + "category": "combined", + "n": 300, + "detected": 299, + "errors": 0, + "errorDescription": null, + "detectionRate": 0.9966666666666667, + "medianMs": 0.39829199999803677, + "p25Ms": 0.1657294999986334, + "p75Ms": 0.8543335000003935, + "medianCases": 1307, + "p75Cases": 2638, + "medianRunLength": 3, + "medianRunSum": 38, + "optimalRate": null, + "censoredMs": 2.864541000000827 + }, + { + "name": "combined/open-payload", + "category": "combined", + "n": 300, + "detected": 286, + "errors": 0, + "errorDescription": null, + "detectionRate": 0.9533333333333334, + "medianMs": 5.418645999997807, + "p25Ms": 2.928343999999015, + "p75Ms": 9.561395999999604, + "medianCases": 2358.5, + "p75Cases": 4392.5, + "medianRunLength": 8, + "medianRunSum": 173, + "optimalRate": null, + "censoredMs": 18.175187999998343 + }, + { + "name": "combined/wide-easy-branch", + "category": "combined", + "n": 300, + "detected": 299, + "errors": 0, + "errorDescription": null, + "detectionRate": 0.9966666666666667, + "medianMs": 0.42129100000056496, + "p25Ms": 0.1757290000023204, + "p75Ms": 0.873145999999906, + "medianCases": 1307, + "p75Cases": 2638, + "medianRunLength": 3, + "medianRunSum": 38, + "optimalRate": null, + "censoredMs": 3.0880830000005517 + }, + { + "name": "deep/big-int", + "category": "deep", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.21627100000023347, + "p25Ms": 0.18139574999986507, + "p75Ms": 0.2626144999994722, + "medianCases": 22, + "p75Cases": 44, + "medianRunLength": 2, + "medianRunSum": 2000015, + "optimalRate": null, + "censoredMs": null + }, + { + "name": "deep/list-sum", + "category": "deep", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 21.58362500000112, + "p25Ms": 17.099624500000118, + "p75Ms": 30.159198249997644, + "medianCases": 3, + "p75Cases": 5, + "medianRunLength": 25, + "medianRunSum": 112, + "optimalRate": null, + "censoredMs": null + }, + { + "name": "deep/long-string", + "category": "deep", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 48.0745205000012, + "p25Ms": 37.533541499999956, + "p75Ms": 57.70166700000118, + "medianCases": 3, + "p75Cases": 6, + "medianRunLength": 64, + "medianRunSum": 33, + "optimalRate": null, + "censoredMs": null + }, + { + "name": "filter/multiple-of-four", + "category": "filter", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.038646000000426284, + "p25Ms": 0.028583749999597785, + "p75Ms": 0.052958499999704145, + "medianCases": 52, + "p75Cases": 98.5, + "medianRunLength": 1, + "medianRunSum": 42, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "intervals/touching-merge", + "category": "known-bug", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.9421875000005002, + "p25Ms": 0.5922812499993597, + "p75Ms": 1.4249999999974534, + "medianCases": 11, + "p75Cases": 22, + "medianRunLength": 7, + "medianRunSum": 2, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "needle/exact-list", + "category": "needle", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.435770500000217, + "p25Ms": 0.24083300000256713, + "p75Ms": 0.7221567500009769, + "medianCases": 481, + "p75Cases": 914.5, + "medianRunLength": 7, + "medianRunSum": 9, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "needle/int-in-range", + "category": "needle", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.10043749999977081, + "p25Ms": 0.048583499999040214, + "p75Ms": 0.1642705000017486, + "medianCases": 330, + "p75Cases": 561, + "medianRunLength": 2, + "medianRunSum": 43, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "needle/pair-diagonal", + "category": "needle", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.02087500000197906, + "p25Ms": 0.014593750000130967, + "p75Ms": 0.03214575000038167, + "medianCases": 33, + "p75Cases": 66.5, + "medianRunLength": 2, + "medianRunSum": 0, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "queue/missing-invariant", + "category": "known-bug", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.5632704999989073, + "p25Ms": 0.5084167499999239, + "p75Ms": 0.6514377499979673, + "medianCases": 17, + "p75Cases": 32.25, + "medianRunLength": 11, + "medianRunSum": 6, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "rbt/missing-rotation", + "category": "known-bug", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.2899790000010398, + "p25Ms": 0.26837475000002087, + "p75Ms": 0.3357192500025121, + "medianCases": 8, + "p75Cases": 16, + "medianRunLength": 7, + "medianRunSum": 6, + "optimalRate": 1, + "censoredMs": null + }, + { + "name": "structural/just-nothing", + "category": "structural", + "n": 300, + "detected": 300, + "errors": 0, + "errorDescription": null, + "detectionRate": 1, + "medianMs": 0.012145499998950982, + "p25Ms": 0.01074999999946158, + "p75Ms": 0.014500000000282398, + "medianCases": 4, + "p75Cases": 7, + "medianRunLength": 2, + "medianRunSum": 1, + "optimalRate": 1, + "censoredMs": null + } + ] +} diff --git a/f-metric/build.sh b/f-metric/build.sh new file mode 100755 index 00000000..d38857ea --- /dev/null +++ b/f-metric/build.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Compiles the F-metric harness. +# +# Like tests/run-tests.sh, this copies the library from ../src into a local +# ELM_HOME so that the harness compiles against *this working copy* of +# elm-explorations/test rather than a published release. (The trick +# benchmarks/ uses -- putting ../src on source-directories -- can't work, +# because applications aren't allowed to import Elm.Kernel.* modules.) +# +# Compiles in dev mode, deliberately and with no option to change it: that is +# what node-test-runner does (lib/ElmCompiler.js only ever passes --output and +# --report), and it has to, because under --optimize Debug.toString returns +# "" for everything, so every counterexample elm-test reports would +# be unreadable. Dev mode is the production configuration here, not a +# compromise. +# +# With NOCLEANUP=1, keeps elm-stuff between builds. Faster, but only safe when +# ../src hasn't changed. +# +# The local ELM_HOME is reused across builds. Delete f-metric/elm_home to +# force a clean one (needs network access to repopulate). + +set -euo pipefail + +cd "${0%/*}" + +PACKAGE_NAME=$(grep '"name"' ../elm.json | cut -d \" -f4) +PACKAGE_VERSION=$(grep '"version"' ../elm.json | cut -d \" -f4) +ELM_HOME_DIR="elm_home" +PACKAGE_PATH="${ELM_HOME_DIR}/0.19.2/packages/${PACKAGE_NAME}/${PACKAGE_VERSION}" + +echo "Copying the library from ../src to ${PACKAGE_PATH}" +rm -rf "${PACKAGE_PATH}" +# Elm caches compiled dependency artifacts per package *version*, and the +# version never changes as we edit ../src, so without this the build silently +# reuses a stale copy of the library. +if [ -z "${NOCLEANUP+x}" ]; then + rm -rf elm-stuff +fi +mkdir -p "${PACKAGE_PATH}" +cp -r ../src "${PACKAGE_PATH}/src" +cp ../elm.json ../README.md ../LICENSE "${PACKAGE_PATH}/" + +echo "Compiling the harness" +ELM_HOME="${ELM_HOME_DIR}" elm make src/Main.elm --output elm.js + +echo "Built f-metric/elm.js" diff --git a/f-metric/elm.json b/f-metric/elm.json new file mode 100644 index 00000000..4652309c --- /dev/null +++ b/f-metric/elm.json @@ -0,0 +1,25 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.2", + "dependencies": { + "direct": { + "elm/core": "1.0.5", + "elm/json": "1.1.3", + "elm/random": "1.0.0", + "elm-explorations/test": "2.2.1" + }, + "indirect": { + "elm/bytes": "1.0.8", + "elm/html": "1.0.0", + "elm/time": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + }, + "test-dependencies": { + "direct": {}, + "indirect": {} + } +} diff --git a/f-metric/runner.mjs b/f-metric/runner.mjs new file mode 100755 index 00000000..8af80ddc --- /dev/null +++ b/f-metric/runner.mjs @@ -0,0 +1,633 @@ +#!/usr/bin/env node + +// Drives the compiled Elm harness (elm.js) and turns raw measurements into a +// report. All statistics live here rather than in Elm, so re-analysing a saved +// run never needs a recompile. +// +// Usage: +// ./build.sh && node runner.mjs [options] +// +// --seeds N measurements per case (default 300) +// --budget N max fuzz runs per measurement (default 10000). +// A case can raise this for itself; both sides of a +// comparison must use the same value. +// --filter REGEX only run cases whose name matches +// --shrink-only also run the cases that measure only simplification +// (falsified by the first generated value, so they say +// nothing about search quality and are off by default) +// --warmup N discarded measurements per case, to let the JIT settle +// (default 3) +// --label NAME recorded in the output, shown in comparisons +// --out FILE write JSONL here (default results/