Dispatch add-auth evals after stable Clerk CLI releases

.github/workflows/release.yml

@@ -118,6 +118,24 @@ jobs:
           ARTIFACTS_DIR: ${{ github.workspace }}/dist/artifacts
           GH_TOKEN: ${{ github.token }}
 
+  dispatch-add-auth-eval:
+    needs: [versioning, publish-npm]
+    if: needs.versioning.outputs.release_created == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: oven-sh/setup-bun@v2
+      - name: Notify clerk-evals of the stable release
+        run: bun scripts/dispatch-evals.ts
+        env:
+          CLERK_CLI_VERSION: ${{ needs.versioning.outputs.version }}
+          CLERK_EVALS_DISPATCH_TOKEN: ${{ secrets.CLERK_EVALS_DISPATCH_TOKEN }}
+
   publish-github:
     needs: [versioning, build, sign-macos, smoke-test, publish-npm]
     runs-on: ubuntu-latest
@@ -175,7 +193,7 @@ jobs:
       always() &&
       needs.versioning.outputs.release_created == 'true' &&
       contains(needs.*.result, 'failure')
-    needs: [versioning, build, sign-macos, smoke-test, publish-npm, publish-github, homebrew]
+    needs: [versioning, build, sign-macos, smoke-test, publish-npm, publish-github, homebrew, dispatch-add-auth-eval]
     uses: ./.github/workflows/notify-failure.yml
     with:
       workflow-name: Stable release

docs/releasing.md

@@ -178,12 +178,15 @@ The releaser accepts these flags:
 
 All publishes are idempotent -- the script checks the registry before publishing and skips already-published versions, and a publish that npm rejects because the version already exists counts as done. The check reads the per-version document (`registry.npmjs.org/<name>/<version>`), which the registry CDN does not cache, but npm can still take several minutes to serve a new version back. Before publishing the `clerk` wrapper, the script waits up to two minutes for every platform package to be readable; if one is still unreadable after that, it logs a warning and publishes the wrapper anyway, because npm already accepted every platform publish. If one platform publish fails, the others still run to completion before the job exits, so a re-run only has to publish what is actually missing.
 
+After a stable npm publish succeeds, `dispatch-add-auth-eval` sends `clerk_cli_released` with that version to `clerk/clerk-evals` without waiting for npm's `latest` tag to catch up. Canary and snapshot releases do not send this event. The job uses `CLERK_EVALS_DISPATCH_TOKEN`, a dedicated Actions secret with Contents: read and write on only `clerk/clerk-evals`; the default `GITHUB_TOKEN` stays read-only. A dispatch failure triggers the stable-release failure notification. The eval receiver waits for the version to become `clerk@latest`, pins a `clerk/skills` SHA, and records both versions with its results; a receiver failure is reported from `clerk-evals`, not from the CLI release workflow.
+
 #### Environment Variables
 
 The releaser script and publish workflow steps use these environment variables:
 
 - `ARTIFACTS_DIR` -- path to directory containing compiled binaries from the build job (defaults to `./dist/artifacts`)
 - `GH_TOKEN` -- GitHub token used by the releaser to create tags and releases (stable only)
+- `CLERK_EVALS_DISPATCH_TOKEN` -- dedicated token for the stable release's `clerk/clerk-evals` repository dispatch (not used by npm publishing)
 
 #### npm Authentication
 

scripts/dispatch-evals.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, test } from "bun:test";
+import { assertStableVersion, sendCliReleaseDispatch } from "./dispatch-evals.ts";
+
+describe("stable CLI eval dispatch", () => {
+  test("rejects canary and snapshot versions", () => {
+    expect(() => assertStableVersion("3.5.0-canary.1")).toThrow("stable");
+    expect(() => assertStableVersion("3.5.0-snapshot.1")).toThrow("stable");
+    expect(() => assertStableVersion("3.5.0")).not.toThrow();
+  });
+
+  test("sends the released version as a repository dispatch", async () => {
+    let request: { url: string; init?: RequestInit } | undefined;
+    await sendCliReleaseDispatch("3.5.0", "test-token", async (url, init) => {
+      request = { url, init };
+      return new Response(null, { status: 204 });
+    });
+    expect(request?.url).toBe("https://api.github.com/repos/clerk/clerk-evals/dispatches");
+    expect(request?.init?.method).toBe("POST");
+    expect(JSON.parse(String(request?.init?.body))).toEqual({
+      event_type: "clerk_cli_released",
+      client_payload: { version: "3.5.0" },
+    });
+  });
+
+  test("requires the dedicated credential and reports API rejection", async () => {
+    await expect(sendCliReleaseDispatch("3.5.0", "")).rejects.toThrow("CLERK_EVALS_DISPATCH_TOKEN");
+    await expect(
+      sendCliReleaseDispatch(
+        "3.5.0",
+        "test-token",
+        async () => new Response('{"message":"Forbidden"}', { status: 403 }),
+      ),
+    ).rejects.toThrow("HTTP 403");
+  });
+});

scripts/dispatch-evals.ts

@@ -0,0 +1,44 @@
+/** Notify the add-auth eval receiver after a stable CLI release is published. */
+
+type Fetch = (input: string, init?: RequestInit) => Promise<Response>;
+
+export function assertStableVersion(version: string): void {
+  if (!/^\d+\.\d+\.\d+$/.test(version)) {
+    throw new Error(`Expected a stable Clerk CLI version, got ${JSON.stringify(version)}`);
+  }
+}
+
+export async function sendCliReleaseDispatch(
+  version: string,
+  token: string,
+  fetchImpl: Fetch = fetch,
+): Promise<void> {
+  assertStableVersion(version);
+  if (!token) throw new Error("CLERK_EVALS_DISPATCH_TOKEN is not configured");
+
+  const response = await fetchImpl("https://api.github.com/repos/clerk/clerk-evals/dispatches", {
+    method: "POST",
+    headers: {
+      accept: "application/vnd.github+json",
+      authorization: `Bearer ${token}`,
+      "content-type": "application/json",
+      "x-github-api-version": "2022-11-28",
+    },
+    body: JSON.stringify({ event_type: "clerk_cli_released", client_payload: { version } }),
+    signal: AbortSignal.timeout(15_000),
+  });
+  if (response.status !== 204) {
+    const detail = (await response.text().catch(() => "")).slice(0, 200);
+    throw new Error(
+      `clerk-evals dispatch returned HTTP ${response.status}${detail ? `: ${detail}` : ""}`,
+    );
+  }
+}
+
+if (import.meta.main) {
+  const version = process.env.CLERK_CLI_VERSION ?? "";
+  const token = process.env.CLERK_EVALS_DISPATCH_TOKEN ?? "";
+  if (!token) throw new Error("CLERK_EVALS_DISPATCH_TOKEN is not configured");
+  await sendCliReleaseDispatch(version, token);
+  console.log(`Sent clerk_cli_released for clerk@${version} to clerk/clerk-evals`);
+}