ci(evals): prune duplicate onboarding eval cases and shard the onboarding suite

evals/README.md

@@ -190,6 +190,22 @@ npm run eval:<suite>
 npm run eval:aggregate               # rebuild eval-scores.json from results
 ```
 
+### Sharded suites (`shards.js`)
+
+A slow suite can run as several CI jobs. Set `shards: N` on its entry in `scripts/_manifest.js`. The "Compute changed suites" job then puts `<suite>#1` to `<suite>#N` in the matrix when the suite changes, and each job runs only its share of the tests. `release-autopilot-onboarding` has `shards: 3`.
+
+The split is deterministic. Tests go, heaviest first by `vars.max_turns` (20 when unset), to the shard with the fewest turns so far, so the shards take about the same time. Adding or removing a test can move other tests to another shard. Sharding selects tests by `description`, so every test in a sharded suite needs a unique description (a unit test checks this for the onboarding suite).
+
+Each shard writes `<suite>/results.shard-<i>-of-<N>.json`. When you aggregate without `--run`, `aggregate.js` merges a suite's shard files into one score, and it fails if a shard is missing. Run a shard locally with:
+
+```bash
+node scripts/aggregate.js --run --only=release-autopilot-onboarding#2   # shard 2 of the manifest's 3
+node scripts/aggregate.js --run --only=launchdarkly-release-plans --shard=1/2  # any suite, any split
+node scripts/aggregate.js --only=release-autopilot-onboarding             # merge the shard files
+```
+
+An unsharded run of a suite deletes its shard files, and a sharded run deletes its `results.json`, so one score never mixes the two.
+
 ### README badge rendering (`render-badges.js`)
 
 `eval:badges` rewrites the eval score block in each skill's README between `<!-- eval-score:start -->` and `<!-- eval-score:end -->` markers:
@@ -278,6 +294,7 @@ Add an entry to `scripts/_manifest.js`:
   skillKey: "<domain>/<skill-name>",   // e.g. "agentcontrol/configs-create"
   skillDir: "skills/<domain>/<skill-name>",
   readme: "skills/<domain>/<skill-name>/README.md",
+  // shards: 2,                         // optional, for a slow suite (see "Sharded suites")
 },
 ```
 

evals/launchdarkly-release-plans/assertions.js

@@ -276,3 +276,26 @@ module.exports.picksDefaultAmongComplete = (output) => {
   const created = steps.filter((t) => t.tool === "create-environment-release-plan").length;
   return result(offered && set && created === 0, `offers both plans: ${offered}; sets gradual as the default: ${set}; plans created: ${created}`);
 };
+
+// A complete plan becomes the default: Gradual is set as the default, and no
+// plan is created.
+module.exports.existingPlanSetAsDefault = (output) => {
+  const steps = traj(output);
+  const created = steps.filter((t) => t.tool === "create-environment-release-plan").length;
+  const def = steps.some((t) => t.tool === "update-default-environment-release-plan" && (t.arguments || {}).releasePlanKey === "gradual" && t.arguments.default === true);
+  return result(created === 0 && def, `plans created: ${created}; existing plan set as default: ${def}`);
+};
+
+// Making another plan the default says which default it replaces, in a
+// question asked before the default changes. The question hides the text
+// above it, so assistant turns and the final reply don't count.
+module.exports.saysReplacesDefault = (output) => {
+  const steps = traj(output);
+  const write = steps.findIndex((t) => t.tool === "update-default-environment-release-plan");
+  const asked = steps.slice(0, write === -1 ? steps.length : write)
+    .filter((t) => t.tool === "ask-question")
+    .map((t) => JSON.stringify(t.arguments || {}))
+    .join("\n");
+  const pass = /replaces\s+[`"*\\]*(fast|fast rollout)[`"*\\]*\s+as\s+the\s+default/i.test(asked);
+  return result(pass, pass ? "the confirmation says the plan replaces Fast as the default" : "no question before the change says the new default replaces the current one");
+};

evals/launchdarkly-release-plans/promptfooconfig.yaml

@@ -577,3 +577,53 @@ tests:
         value: file://../launchdarkly-release-plans/assertions.js:confirmsBeforeWrite
         metric: confirms_before_write
         weight: 3
+
+  # ------------------------------------------------------------------
+  # Production's default is Fast; the user makes Gradual the default
+  # instead. The confirmation says Gradual replaces Fast as the default,
+  # Gradual is set as the default, and no plan is created. (Moved from the
+  # onboarding suite, where the wizard reached the same step.)
+  # ------------------------------------------------------------------
+  - description: "Another plan as default: says it replaces the current default, sets it, creates nothing"
+    vars:
+      max_turns: 16
+      mock_ask_question_match: true
+      user_request: "Make Gradual the default release plan for production in LaunchDarkly project default."
+      mock_overrides:
+        get-environment-release-plans:
+          by_arg: environmentKey
+          cases:
+            production:
+              items:
+                - key: fast
+                  name: Fast
+                  environmentKey: production
+                  default: true
+                  progressiveRelease:
+                    randomizationUnit: user
+                    advancementMode: automatic
+                    cleanupBehavior: remove_rules_and_update_default_rule
+                    phases:
+                      - clauses: [{ attribute: kind, op: in, values: ["user"], contextKind: user }]
+                        steps: [{ allocation: 50000, durationMillis: 3600000 }]
+                    metrics: {}
+                - { key: gradual, name: Gradual, environmentKey: production, default: false, progressiveRelease: *progressive }
+              totalCount: 2
+          default: { items: [], totalCount: 0 }
+      mock_ask_question_answers:
+        - ["Use this plan"]
+        - ["Gradual"]
+        - ["Make this change"]
+    assert:
+      - type: javascript
+        value: file://../launchdarkly-release-plans/assertions.js:existingPlanSetAsDefault
+        metric: existing_plan_default
+        weight: 4
+      - type: javascript
+        value: file://../launchdarkly-release-plans/assertions.js:saysReplacesDefault
+        metric: says_replaces_default
+        weight: 3
+      - type: javascript
+        value: file://../launchdarkly-release-plans/assertions.js:confirmsBeforeWrite
+        metric: confirms_before_write
+        weight: 3

evals/release-autopilot-onboarding/assertions.js

@@ -541,12 +541,6 @@ module.exports.stopsReleaseSetupOnPlanError = (output) => {
   return result(policies === 0 && creates === 0 && !releasingOn, `policy reads: ${policies}; plans created: ${creates}; auto-releasing turned on for a repo: ${releasingOn}`);
 };
 
-// Making another plan the default says which default it replaces.
-module.exports.saysReplacesDefault = (output) => {
-  const pass = /replaces\s+[`"*]*(fast|fast rollout)[`"*]*\s+as\s+the\s+default/i.test([allText(output), ...askPrompts(output)].join("\n"));
-  return result(pass, pass ? "says the plan replaces Fast as the default" : "never says the new default replaces the current one");
-};
-
 // Verification confirms GitHub works, names each configured repo, and offers
 // a test pull request.
 module.exports.verifyReportsSetup = (output) => {
@@ -566,15 +560,6 @@ module.exports.cleanupKeepsExistingRepos = (output) => {
   return result(pass, `cleanup repositories written: ${JSON.stringify(repos)}`);
 };
 
-// An environment with a complete plan the user keeps: set it as default, do
-// not create another.
-module.exports.existingPlanSetAsDefault = (output) => {
-  const steps = traj(output);
-  const created = steps.filter((t) => t.tool === "create-environment-release-plan").length;
-  const def = steps.some((t) => t.tool === "update-default-environment-release-plan" && (t.arguments || {}).releasePlanKey === "gradual" && t.arguments.default === true);
-  return result(created === 0 && def, `plans created: ${created}; existing plan set as default: ${def}`);
-};
-
 // Archival already on: nothing to do for it.
 module.exports.noArchivalStep = (output) => {
   const steps = traj(output);
@@ -584,12 +569,6 @@ module.exports.noArchivalStep = (output) => {
   return result(asked === 0 && writes === 0, `archival browser questions: ${asked}; archival writes: ${writes}`);
 };
 
-module.exports.reportListsDefaultPlan = (output) => {
-  const text = finalText(output).toLowerCase();
-  const pass = /default/.test(text) && /progressive/.test(text) && /production/.test(text);
-  return result(pass, pass ? "report names production's default plan" : "report does not name the default plan");
-};
-
 module.exports.reportListsCleanup = (output) => {
   const text = finalText(output).toLowerCase();
   const pass = /cleanup/.test(text) && /weekly/.test(text) && text.includes("acme/demo");
@@ -718,31 +697,6 @@ module.exports.followUpHasFlagReleasePlanCommand = (output) => {
   return result(beta && !asked, `flag release plan commands in the report: ${cmds.length}; all with the beta header: ${beta}; asks for the token: ${asked}`);
 };
 
-// Preview lists the flag release plan, writes nothing, and asks the user to
-// run nothing.
-module.exports.previewListsFlagReleasePlan = (output) => {
-  const listed = /flag release plan/i.test(finalText(output));
-  const asked = traj(output).filter((t) => t.tool === "ask-question" && /run this command/i.test(JSON.stringify(t.arguments || {}))).length;
-  const w = writes(output).length;
-  return result(listed && asked === 0 && w === 0, `lists the flag release plan: ${listed}; asks to run a command: ${asked}; writes: ${w}`);
-};
-
-// The project's default flag release plan does not match: after "Looks
-// good", the replace command keeps its key, the default on, its version, and
-// production's release plan. No create command.
-module.exports.flagReleasePlanUpdatedInPlace = (output) => {
-  const steps = traj(output);
-  const put = checks.shownCommands(output).find((c) => c.method === "PUT" && /\/flag-release-plans\/staged-rollout$/.test(c.path));
-  const posts = checks.planCreates(output).length;
-  if (!put || !put.body) return result(false, `no replace command for staged-rollout; create commands: ${posts}`);
-  const b = put.body;
-  const envs = (b.environments || []).map((e) => `${e.environmentKey}:${e.environmentReleasePlanKey || "immediate"}`);
-  const ci = steps.findIndex(isConfirmQuestion);
-  const after = ci !== -1 && put.turn > steps[ci].turn && approvedConfirmBefore(steps, ci + 1) !== -1;
-  const pass = b.default === true && b.version === 2 && JSON.stringify(envs) === JSON.stringify(["production:gradual"]) && after && posts === 0;
-  return result(pass, `default ${b.default}; version ${b.version}; environments ${JSON.stringify(envs)}; after "Looks good": ${after}; create commands: ${posts}`);
-};
-
 // The flag release plan is a follow-up, and auto-releasing stays on as the
 // user picked: the repo write turns it on, and no write turns it off.
 module.exports.autoReleasingOnWithFollowUp = (output) => {