From 40648e4cf7d842fdb0242ac4a4b782c8761a3589 Mon Sep 17 00:00:00 2001 From: JustKira Date: Fri, 20 Nov 2026 13:56:10 +0200 Subject: [PATCH 1/8] Add Tokamak CLI and local plugin index --- bun.lock | 5 + package.json | 4 +- packages/build/src/build.ts | 5 +- packages/build/src/descriptor.ts | 14 +- packages/build/src/vite.ts | 13 +- packages/cli/README.md | 63 ++++++++ packages/cli/package.json | 5 + packages/cli/src/command.ts | 17 ++ packages/cli/src/commands.ts | 7 + packages/cli/src/commands/add.ts | 28 ++++ packages/cli/src/commands/build.ts | 54 +++++++ packages/cli/src/commands/list.ts | 26 +++ packages/cli/src/commands/remove.ts | 18 +++ packages/cli/src/data.ts | 20 +++ packages/cli/src/main.ts | 72 +++++++++ packages/cli/src/plugin-index.ts | 36 +++++ packages/cli/src/server.ts | 60 +++++++ packages/cli/tests/cli.test.mjs | 211 +++++++++++++++++++++++++ packages/cli/tsconfig.json | 4 + packages/server/src/backend.rs | 11 +- packages/server/src/main.rs | 13 +- packages/server/src/plugins/catalog.rs | 83 ++++++++-- packages/server/src/plugins/mod.rs | 2 + packages/server/src/service/startup.rs | 12 +- packages/server/tests/plugins.rs | 107 +++++++++++++ 25 files changed, 861 insertions(+), 29 deletions(-) create mode 100644 packages/cli/README.md create mode 100644 packages/cli/package.json create mode 100644 packages/cli/src/command.ts create mode 100644 packages/cli/src/commands.ts create mode 100644 packages/cli/src/commands/add.ts create mode 100644 packages/cli/src/commands/build.ts create mode 100644 packages/cli/src/commands/list.ts create mode 100644 packages/cli/src/commands/remove.ts create mode 100644 packages/cli/src/data.ts create mode 100644 packages/cli/src/main.ts create mode 100644 packages/cli/src/plugin-index.ts create mode 100644 packages/cli/src/server.ts create mode 100644 packages/cli/tests/cli.test.mjs create mode 100644 packages/cli/tsconfig.json diff --git a/bun.lock b/bun.lock index dc1f6779..da171228 100644 --- a/bun.lock +++ b/bun.lock @@ -58,6 +58,9 @@ "@types/node": "26.6.6", }, }, + "packages/cli": { + "name": "@tokamak/cli", + }, "packages/runtime": { "name": "@tokamak/runtime", "dependencies": { @@ -600,6 +603,8 @@ "@tokamak/build": ["@tokamak/build@workspace:packages/build"], + "@tokamak/cli": ["@tokamak/cli@workspace:packages/cli"], + "@tokamak/runtime": ["@tokamak/runtime@workspace:packages/runtime"], "@tokamak/sdk": ["@tokamak/sdk@workspace:packages/sdk"], diff --git a/package.json b/package.json index 70f8687c..543bab67 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "workspaces": [ "app", "packages/build", + "packages/cli", "packages/sdk", "packages/runtime", "packages/ui", @@ -18,11 +19,12 @@ "e2e": "moon run :e2e", "app": "moon run app:run", "app:dev": "moon run app:run", + "tokamak": "bun packages/cli/src/main.ts", "plugin:build": "proto run bun -- packages/build/src/build.ts", "server:start": "moon run app:server", "server:stop": "proto run bun -- packages/runtime/tools/run-app.ts stop debug", "server:status": "proto run bun -- packages/runtime/tools/run-app.ts status debug", - "typecheck": "moon run sdk:check build:check runtime:ts-check ui:check app:check workspace:plugins-check workspace:e2e-check workspace:wiki-check", + "typecheck": "proto run bun -- check --no-script --project packages/cli/tsconfig.json && moon run sdk:check build:check runtime:ts-check ui:check app:check workspace:plugins-check workspace:e2e-check workspace:wiki-check", "e2e:browser": "proto run bun -- e2e/browser.ts" }, "devDependencies": { diff --git a/packages/build/src/build.ts b/packages/build/src/build.ts index 59d62e47..85da0517 100644 --- a/packages/build/src/build.ts +++ b/packages/build/src/build.ts @@ -6,6 +6,8 @@ import { dirname, resolve } from "node:path"; import { tokamak } from "./vite"; const args = process.argv.slice(2); +const reportResult = args[0] === "--result-json"; +if (reportResult) args.shift(); if ( args.length > 2 || (args.length === 2 && args[0] !== "--config") || @@ -69,4 +71,5 @@ for (const file of files) { } changed++; } -console.log(`Plugin artifact: ${changed} changed, ${unchanged} unchanged`); +if (reportResult) console.log(`Tokamak build result: ${JSON.stringify({ outputFolder: directory })}`); +else console.log(`Plugin artifact: ${changed} changed, ${unchanged} unchanged`); diff --git a/packages/build/src/descriptor.ts b/packages/build/src/descriptor.ts index 0c1bf62a..8f84aa2f 100644 --- a/packages/build/src/descriptor.ts +++ b/packages/build/src/descriptor.ts @@ -3,8 +3,9 @@ import { readPluginCompilerOptions } from "../../sdk/tools/plugin-boundary.mjs"; import { isServerModule } from "../../sdk/tools/server-files.mjs"; import ts from "typescript"; import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { readFile } from "node:fs/promises"; import { basename, relative, resolve, sep } from "node:path"; -import type { PluginDefinition } from "@tokamak/sdk"; +import { definePlugin, type PluginDefinition } from "@tokamak/sdk"; export function serverDirectory(root: string): string { return resolve(root, existsSync(resolve(root, "src/server")) ? "src/server" : "backend"); @@ -88,6 +89,17 @@ export function readDescriptor(root: string): PluginDefinition | undefined { return value as unknown as PluginDefinition; } +export async function readSourcePlugin(root: string): Promise { + const descriptor = readDescriptor(root); + if (descriptor) return definePlugin(descriptor); + const manifest = JSON.parse(await readFile(resolve(root, "package.json"), "utf8")); + if (typeof manifest.name !== "string") throw new Error("Plugin package.json must name the plugin"); + return definePlugin({ + id: manifest.name.replace(/^@/, "").replaceAll("/", "."), + version: manifest.version ?? "0.1.0", + }); +} + /** The existing backend protocol has three public actions. Derive its provider * from those exports; derive consumers from literal ctx.providers calls. */ export function deriveInterfaces(root: string) { diff --git a/packages/build/src/vite.ts b/packages/build/src/vite.ts index 75fbe47c..76b19799 100644 --- a/packages/build/src/vite.ts +++ b/packages/build/src/vite.ts @@ -13,7 +13,7 @@ import { clientSurface } from "./client-surface.ts"; import { uiBuildConfig, clientAssets } from "./ui-build-config.ts"; import { publicFunctions } from "./exports.ts"; import { - readDescriptor, + readSourcePlugin, serverDirectory, deriveInterfaces, deriveProviders, @@ -73,16 +73,7 @@ export function tokamak( }, async buildStart() { checkPluginImports(root); - const descriptor = options.plugin ?? readDescriptor(root); - const manifest = descriptor - ? undefined - : JSON.parse(await readFile(resolve(root, "package.json"), "utf8")); - plugin = definePlugin( - descriptor ?? { - id: String(manifest.name).replace(/^@/, "").replaceAll("/", "."), - version: manifest.version ?? "0.1.0", - }, - ); + plugin = options.plugin ? definePlugin(options.plugin) : await readSourcePlugin(root); const server = serverDirectory(root); if (existsSync(resolve(root, "plugin.ts"))) { plugin = definePlugin({ diff --git a/packages/cli/README.md b/packages/cli/README.md new file mode 100644 index 00000000..2f76d852 --- /dev/null +++ b/packages/cli/README.md @@ -0,0 +1,63 @@ +# Tokamak CLI + +Run from a checkout with the pinned Bun: + +```sh +proto run bun -- run tokamak --help +proto run bun -- run tokamak plugin build /path/to/my-plugin --json +proto run bun -- run tokamak plugin add /path/to/my-plugin +proto run bun -- run tokamak plugin list --json +proto run bun -- run tokamak plugin remove my-plugin +``` + +`build` runs the artifact build once, defaults to the current folder, and +doesn't need a server. `add` reads the plugin's static identity from `plugin.ts` +(or the build's `package.json` convention), and doesn't build it. `add` and +`remove` only edit the index; the local server picks up membership changes on +its next start. + +The app data folder contains `plugin-index.json`, a JSON object mapping each +plugin id to its absolute source folder, for example: + +```json +{ "my-plugin": "/path/to/my-plugin" } +``` + +At startup the local server loads each indexed folder's `dist` artifact in +addition to its `--plugins` folder. A built indexed plugin wins when both +sources contain the same id. A missing index is empty; missing source folders +and entries without a build are logged and skipped. + +`list` asks the running local server for its catalog. The CLI discovers it through +`server.json` and authenticates with `server.token` in the app data folder. Set +`TOKAMAK_SERVER_DIR` to use a different app data folder, including for tests. +For a server at a specific address, set `TOKAMAK_SERVER` and `TOKAMAK_TOKEN`, +or supply `--server
` with `TOKAMAK_TOKEN`. The flag takes precedence. +The CLI never starts a server. + +With `--json`, stdout contains one JSON value and human text goes to stderr. +Without it, human text goes to stdout. Exit status is zero on success and +nonzero on failure. The CLI never prompts. Success has `ok: true` and: + +| Command | Additional result fields | +| --------------- | --------------------------------------------------- | +| `plugin build` | `id`, `outputFolder` | +| `plugin add` | `id`, `folder`, `restartRequired: true` | +| `plugin remove` | `id`, `restartRequired: true` | +| `plugin list` | `plugins`: an array of `{ id, version, fromIndex }` | +| `--help` | `help` | + +Failures return `{ ok: false, error: { message, next } }`. Build failures with +TypeScript locations also return `error.diagnostics`, an array of +`{ file, line, column, code, message }`. + +To add a command, create a file under `src/commands/` that satisfies `Command`, +then register it in `src/commands.ts`. The entry point handles flags, help, +output and exit codes. The CLI test walks the registry to check that contract. + +Run the CLI tests with the real server: + +```sh +cargo build -p server --bin tokamak-server +proto run bun -- test packages/cli/tests/cli.test.mjs +``` diff --git a/packages/cli/package.json b/packages/cli/package.json new file mode 100644 index 00000000..e37c4f89 --- /dev/null +++ b/packages/cli/package.json @@ -0,0 +1,5 @@ +{ + "name": "@tokamak/cli", + "private": true, + "type": "module" +} diff --git a/packages/cli/src/command.ts b/packages/cli/src/command.ts new file mode 100644 index 00000000..1e43140c --- /dev/null +++ b/packages/cli/src/command.ts @@ -0,0 +1,17 @@ +export interface Command { + path: string; + summary: string; + example: string; + args?: { name: string; optional?: boolean }; + run(args: string[], options: { server?: string }): Promise<{ data: object; text: string }>; +} + +export class CommandError extends Error { + constructor( + message: string, + readonly next: string, + readonly diagnostics?: object[], + ) { + super(message); + } +} diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts new file mode 100644 index 00000000..c914d9fc --- /dev/null +++ b/packages/cli/src/commands.ts @@ -0,0 +1,7 @@ +import type { Command } from "./command"; +import build from "./commands/build"; +import add from "./commands/add"; +import remove from "./commands/remove"; +import list from "./commands/list"; + +export const commands: readonly Command[] = [build, add, remove, list]; diff --git a/packages/cli/src/commands/add.ts b/packages/cli/src/commands/add.ts new file mode 100644 index 00000000..c961c666 --- /dev/null +++ b/packages/cli/src/commands/add.ts @@ -0,0 +1,28 @@ +import { realpath, stat } from "node:fs/promises"; +import { readSourcePlugin } from "../../../build/src/descriptor"; +import { CommandError, type Command } from "../command"; +import { readIndex, writeIndex } from "../plugin-index"; + +export default { + path: "plugin add", + summary: "Add a source folder to the local plugin index.", + args: { name: "folder" }, + example: "tokamak plugin add ./my-plugin", + async run([source]) { + const folder = await realpath(source!); + if (!(await stat(folder)).isDirectory()) throw new Error(`Not a source folder: ${folder}`); + const { id } = await readSourcePlugin(folder); + const entries = await readIndex(); + const existing = entries.get(id); + if (existing !== undefined && existing !== folder) + throw new CommandError(`Plugin ${id} is already indexed at ${existing}`, `tokamak plugin remove ${id}`); + if (!existing) { + entries.set(id, folder); + await writeIndex(entries); + } + return { + data: { id, folder, restartRequired: true }, + text: `Indexed ${id} at ${folder}. The local server picks this up at its next start.`, + }; + }, +} satisfies Command; diff --git a/packages/cli/src/commands/build.ts b/packages/cli/src/commands/build.ts new file mode 100644 index 00000000..b5972801 --- /dev/null +++ b/packages/cli/src/commands/build.ts @@ -0,0 +1,54 @@ +import { execFile } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { CommandError, type Command } from "../command"; + +const builder = fileURLToPath(new URL("../../../build/src/build.ts", import.meta.url)); + +function diagnostics(message: string) { + const plain = /^(.*)\((\d+),(\d+)\): error (TS\d+): (.*)$/gm; + const tagged = /\s*([^<]*)/g; + return [...message.matchAll(plain), ...message.matchAll(tagged)].map( + ([, file, line, column, code, text]) => ({ + file, + line: Number(line), + column: Number(column), + code, + message: text?.trim(), + }), + ); +} + +export default { + path: "plugin build", + summary: "Build a plugin once.", + args: { name: "folder", optional: true }, + example: "tokamak plugin build . --json", + async run([folder]) { + const { stdout } = await promisify(execFile)( + process.execPath, + [builder, "--result-json", resolve(folder ?? ".")], + { maxBuffer: 16 * 1024 * 1024 }, + ).catch((error: { stdout?: string; stderr?: string; message: string }) => { + const message = [error.stdout, error.stderr].filter(Boolean).join("\n").trim() || error.message; + const details = diagnostics(message); + throw new CommandError( + `Plugin build failed: ${message}`, + "tokamak plugin build --help", + details.length ? details : undefined, + ); + }); + const result = stdout + .split("\n") + .reverse() + .find((line) => line.startsWith("Tokamak build result: ")); + if (!result) throw new Error("The build did not report its output folder"); + const { outputFolder } = JSON.parse(result.slice("Tokamak build result: ".length)); + const { + plugin: { id }, + } = JSON.parse(await readFile(resolve(outputFolder, "manifest.json"), "utf8")); + return { data: { id, outputFolder }, text: `Built ${id}: ${outputFolder}` }; + }, +} satisfies Command; diff --git a/packages/cli/src/commands/list.ts b/packages/cli/src/commands/list.ts new file mode 100644 index 00000000..ed6c1c30 --- /dev/null +++ b/packages/cli/src/commands/list.ts @@ -0,0 +1,26 @@ +import type { Command } from "../command"; +import { serverGet } from "../server"; + +type LoadedPlugin = { plugin: { id: string; version: string }; fromIndex?: boolean }; + +export default { + path: "plugin list", + summary: "List plugins loaded by the running server.", + example: "tokamak plugin list --json", + async run(_, { server }) { + const catalog = (await serverGet("/plugins", server)) as { plugins: LoadedPlugin[] }; + const plugins = catalog.plugins.map(({ plugin: { id, version }, fromIndex }) => ({ + id, + version, + fromIndex: fromIndex === true, + })); + return { + data: { plugins }, + text: plugins.length + ? plugins + .map(({ id, version, fromIndex }) => `${id}\t${version}${fromIndex ? "\tindex" : ""}`) + .join("\n") + : "No plugins loaded.", + }; + }, +} satisfies Command; diff --git a/packages/cli/src/commands/remove.ts b/packages/cli/src/commands/remove.ts new file mode 100644 index 00000000..ff285181 --- /dev/null +++ b/packages/cli/src/commands/remove.ts @@ -0,0 +1,18 @@ +import type { Command } from "../command"; +import { readIndex, writeIndex } from "../plugin-index"; + +export default { + path: "plugin remove", + summary: "Remove a plugin from the local plugin index.", + args: { name: "id" }, + example: "tokamak plugin remove my-plugin", + async run([id]) { + const entries = await readIndex(); + if (!entries.delete(id!)) throw new Error(`Unknown indexed plugin: ${id}`); + await writeIndex(entries); + return { + data: { id, restartRequired: true }, + text: `Removed ${id} from the index. The local server picks this up at its next start.`, + }; + }, +} satisfies Command; diff --git a/packages/cli/src/data.ts b/packages/cli/src/data.ts new file mode 100644 index 00000000..544fb94f --- /dev/null +++ b/packages/cli/src/data.ts @@ -0,0 +1,20 @@ +import { homedir } from "node:os"; +import { isAbsolute, join, resolve } from "node:path"; + +export function dataFolder() { + if (process.env.TOKAMAK_SERVER_DIR) return resolve(process.env.TOKAMAK_SERVER_DIR); + const home = homedir(); + if (process.platform === "darwin") { + return join(home, "Library", "Application Support", "com.pixelscortex.Tokamak"); + } + if (process.platform === "win32") { + return join( + process.env.LOCALAPPDATA || join(home, "AppData", "Local"), + "pixelscortex", + "Tokamak", + "data", + ); + } + const xdg = process.env.XDG_DATA_HOME; + return join(xdg && isAbsolute(xdg) ? xdg : join(home, ".local", "share"), "tokamak"); +} diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts new file mode 100644 index 00000000..216b8cc2 --- /dev/null +++ b/packages/cli/src/main.ts @@ -0,0 +1,72 @@ +import { CommandError } from "./command"; +import { commands } from "./commands"; + +const argv = process.argv.slice(2); +const json = argv.includes("--json"); +const help = argv.includes("--help"); +const positional: string[] = []; +let server: string | undefined; +let next = "tokamak --help"; + +function output(data: object, human: string) { + if (json) { + console.log(JSON.stringify(data)); + console.error(human); + } else { + console.log(human); + } +} + +try { + for (const [i, arg] of argv.entries()) { + if (arg === "--server") { + if (server || !argv[i + 1] || argv[i + 1]!.startsWith("-")) + throw new CommandError("Expected one address after --server", next); + server = argv[i + 1]; + } else if (argv[i - 1] === "--server" || arg === "--help" || arg === "--json") { + continue; + } else if (arg.startsWith("-")) { + throw new CommandError(`Unknown flag: ${arg}`, next); + } else { + positional.push(arg); + } + } + const command = commands.find(({ path }) => path.split(" ").every((word, i) => positional[i] === word)); + if (command) next = `tokamak ${command.path} --help`; + const args = command ? positional.slice(command.path.split(" ").length) : []; + const group = positional.join(" "); + const children = commands.filter(({ path }) => !group || path.startsWith(group + " ")); + if (command && args.length > Number(Boolean(command.args))) { + throw new CommandError(`Unexpected argument: ${args.at(-1)}`, next); + } + if (help && (command || children.length)) { + const text = command + ? [ + `Usage: tokamak ${command.path}${command.args ? (command.args.optional ? ` [${command.args.name}]` : ` <${command.args.name}>`) : ""} [--json] [--server
] [--help]`, + command.summary, + `Example: ${command.example}`, + ].join("\n") + : [ + `Usage: tokamak ${group ? group + " " : ""} [--json] [--server
] [--help]`, + ...children.map(({ path, summary }) => ` ${path} ${summary}`), + `Example: ${children[0]!.example}`, + ].join("\n"); + output({ ok: true, help: text }, text); + } else { + if (!command) + throw new CommandError(group ? `Unknown or incomplete command: ${group}` : "Missing command", next); + if (command.args && !command.args.optional && !args.length) + throw new CommandError(`Missing argument: ${command.args.name}`, next); + const { data, text } = await command.run(args, { server }); + output({ ok: true, ...data }, text); + } +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (error instanceof CommandError) next = error.next; + const diagnostics = error instanceof CommandError ? error.diagnostics : undefined; + output( + { ok: false, error: { message, next, ...(diagnostics ? { diagnostics } : {}) } }, + `${message}\nTry: ${next}`, + ); + process.exitCode = 1; +} diff --git a/packages/cli/src/plugin-index.ts b/packages/cli/src/plugin-index.ts new file mode 100644 index 00000000..02da1f4d --- /dev/null +++ b/packages/cli/src/plugin-index.ts @@ -0,0 +1,36 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { isAbsolute, join } from "node:path"; +import { dataFolder } from "./data"; + +const name = "plugin-index.json"; + +export async function readIndex(): Promise> { + const file = join(dataFolder(), name); + const source = await readFile(file, "utf8").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return "{}"; + throw error; + }); + const parsed: unknown = JSON.parse(source); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) + throw new Error(`Invalid plugin index: ${file}`); + const entries = new Map(); + for (const [id, folder] of Object.entries(parsed)) { + if (typeof folder !== "string" || !isAbsolute(folder)) + throw new Error(`Invalid folder for ${id} in ${file}`); + entries.set(id, folder); + } + return entries; +} + +export async function writeIndex(entries: ReadonlyMap) { + const folder = dataFolder(); + await mkdir(folder, { recursive: true, mode: 0o700 }); + const scratch = join(folder, `.${name}.${randomUUID()}.tmp`); + try { + await writeFile(scratch, JSON.stringify(Object.fromEntries(entries), null, 2) + "\n", { mode: 0o600 }); + await rename(scratch, join(folder, name)); + } finally { + await rm(scratch, { force: true }); + } +} diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts new file mode 100644 index 00000000..d6c46616 --- /dev/null +++ b/packages/cli/src/server.ts @@ -0,0 +1,60 @@ +import { lstat, readFile } from "node:fs/promises"; +import { isIP } from "node:net"; +import { join } from "node:path"; +import { CommandError } from "./command"; +import { dataFolder } from "./data"; + +function httpAddress(address: string) { + const url = new URL(address.includes("://") ? address : `http://${address}`); + if (url.protocol === "ws:") url.protocol = "http:"; + if (url.protocol === "wss:") url.protocol = "https:"; + if (!["http:", "https:"].includes(url.protocol) || url.username || url.password) + throw new Error("Expected an HTTP or HTTPS server address without credentials"); + return url; +} + +async function localEndpoint() { + const folder = dataFolder(); + const { url: address } = JSON.parse(await readFile(join(folder, "server.json"), "utf8")); + const ws = new URL(address); + if ( + ws.protocol !== "ws:" || + ws.pathname !== "/ws" || + ws.search || + ws.hash || + !(ws.hostname === "[::1]" || (isIP(ws.hostname) === 4 && ws.hostname.startsWith("127."))) + ) + throw new Error("Invalid local server discovery address"); + const file = join(folder, "server.token"); + const metadata = await lstat(file); + if (!metadata.isFile() || (process.platform !== "win32" && (metadata.mode & 0o077) !== 0)) + throw new Error("Local server token must be a private regular file"); + const token = await readFile(file, "utf8"); + if (!/^[a-f0-9]{64}$/i.test(token)) throw new Error("Invalid local server token"); + return { url: httpAddress(address), token }; +} + +export async function serverGet(path: string, explicit?: string): Promise { + const address = explicit ?? process.env.TOKAMAK_SERVER; + const next = address ? "tokamak --help" : "tokamak server start"; + try { + const endpoint = address + ? { url: httpAddress(address), token: process.env.TOKAMAK_TOKEN } + : await localEndpoint(); + if (!endpoint.token) throw new Error("Set TOKAMAK_TOKEN for the selected server"); + const response = await fetch(new URL(path, endpoint.url), { + headers: { Authorization: `Bearer ${endpoint.token}` }, + signal: AbortSignal.timeout(5000), + redirect: "error", + }); + if (!response.ok) throw new Error(`Server returned HTTP ${response.status}`); + return await response.json(); + } catch (error) { + if (!address && (error as NodeJS.ErrnoException).code === "ENOENT") + throw new CommandError("No local server is running", next); + throw new CommandError( + `Could not read ${path} from the ${address ? "selected server" : "local server"}: ${error instanceof Error ? error.message : String(error)}`, + next, + ); + } +} diff --git a/packages/cli/tests/cli.test.mjs b/packages/cli/tests/cli.test.mjs new file mode 100644 index 00000000..88dd2b06 --- /dev/null +++ b/packages/cli/tests/cli.test.mjs @@ -0,0 +1,211 @@ +import { expect, test } from "bun:test"; +import { chmod, cp, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { randomBytes } from "node:crypto"; +import { withPlugin, workspace } from "../../build/tests/outside-fixture.mjs"; + +function cli(root, args, extra = {}) { + const result = Bun.spawnSync([process.execPath, "run", "tokamak", ...args], { + cwd: workspace, + env: { + ...process.env, + TOKAMAK_SERVER: undefined, + TOKAMAK_TOKEN: undefined, + TOKAMAK_SERVER_DIR: join(root, "data"), + ...extra, + }, + stdout: "pipe", + stderr: "pipe", + }); + return { code: result.exitCode, stdout: result.stdout.toString(), stderr: result.stderr.toString() }; +} + +test("builds an external plugin and reports its output folder", async () => { + await withPlugin((plugin, root) => { + const result = cli(root, ["plugin", "build", plugin, "--json"]); + expect(result.code, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + ok: true, + id: "outside-fixture", + outputFolder: join(plugin, "dist"), + }); + }); +}, 30_000); + +test("returns structured type errors in both checker formats", async () => { + await withPlugin(async (plugin, root) => { + await writeFile(join(plugin, "src/server/broken.ts"), "\nexport const broken: string = 42;\n"); + for (const AI_AGENT of [undefined, "1"]) { + const result = cli(root, ["plugin", "build", plugin, "--json"], { AI_AGENT }); + expect(result.code).not.toBe(0); + const { error } = JSON.parse(result.stdout); + expect(error.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ file: expect.stringMatching(/broken\.ts$/), line: 2, code: "TS2322" }), + ]), + ); + } + }); +}, 30_000); + +test("adds and removes folders and protects an existing id", async () => { + await withPlugin(async (plugin, root) => { + const index = join(root, "data", "plugin-index.json"); + const add = (folder) => cli(root, ["plugin", "add", folder, "--json"]); + for (let i = 0; i < 2; i++) { + const result = add(plugin); + expect(result.code, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + ok: true, + id: "outside-fixture", + restartRequired: true, + }); + } + expect(JSON.parse(await readFile(index, "utf8"))).toEqual({ "outside-fixture": plugin }); + const other = join(root, "other"); + await cp(plugin, other, { recursive: true }); + const duplicate = add(other); + expect(duplicate.code).not.toBe(0); + expect(JSON.parse(duplicate.stdout).error.next).toBe("tokamak plugin remove outside-fixture"); + const remove = () => cli(root, ["plugin", "remove", "outside-fixture", "--json"]); + const removed = remove(); + expect(removed.code, removed.stderr).toBe(0); + expect(JSON.parse(removed.stdout)).toMatchObject({ + ok: true, + id: "outside-fixture", + restartRequired: true, + }); + expect(JSON.parse(await readFile(index, "utf8"))).toEqual({}); + expect(remove().code).not.toBe(0); + }); +}); + +test("tells the user how to start a missing local server", async () => { + await withPlugin((_, root) => { + const result = cli(root, ["plugin", "list"]); + expect(result.code).not.toBe(0); + expect(result.stdout).toContain("tokamak server start"); + }); +}); + +test("every registered command shares help and JSON output rules", async () => { + const { commands } = await import("../src/commands"); + await withPlugin((_, root) => { + expect(commands.map(({ path }) => path)).toEqual( + expect.arrayContaining(["plugin build", "plugin add", "plugin remove", "plugin list"]), + ); + for (const path of ["", "plugin", ...commands.map((command) => command.path)]) { + const words = path ? path.split(" ") : []; + const shown = cli(root, [...words, "--help"]); + expect(shown.code, shown.stderr).toBe(0); + expect(shown.stdout).toContain("Example: tokamak "); + const success = cli(root, [...words, "--json", "--help"]); + expect(success.code, success.stderr).toBe(0); + expect(JSON.parse(success.stdout)).toEqual({ ok: true, help: shown.stdout.trimEnd() }); + expect(success.stderr).toContain("Example: tokamak "); + const failure = cli(root, [...words, "--json", "--unknown-test-flag"]); + expect(failure.code).not.toBe(0); + expect(JSON.parse(failure.stdout)).toEqual({ + ok: false, + error: { message: expect.stringContaining("Unknown flag"), next: expect.stringContaining("--help") }, + }); + } + for (const command of commands.filter(({ args }) => args && !args.optional)) { + const missing = cli(root, [...command.path.split(" "), "--json"]); + expect(missing.code).not.toBe(0); + expect(JSON.parse(missing.stdout).error.message).toContain("Missing argument"); + } + const unknown = cli(root, ["unknown-command", "--json"]); + expect(unknown.code).not.toBe(0); + expect(JSON.parse(unknown.stdout).error.next).toBe("tokamak --help"); + }); +}, 30_000); + +async function withServer(root, check) { + const data = join(root, "data"); + const plugins = join(root, "installed"); + await mkdir(plugins, { recursive: true }); + await mkdir(data, { recursive: true }); + const token = randomBytes(32).toString("hex"); + const tokenFile = join(data, "server.token"); + await writeFile(tokenFile, token); + if (process.platform !== "win32") await chmod(tokenFile, 0o600); + const target = resolve(workspace, process.env.CARGO_TARGET_DIR ?? "target"); + const binary = join( + target, + "debug", + process.platform === "win32" ? "tokamak-server.exe" : "tokamak-server", + ); + const server = Bun.spawn([binary, "--local-dir", data, "--plugins", plugins], { + cwd: root, + env: { ...process.env, TOKAMAK_WEB_DIR: undefined }, + stdout: "ignore", + stderr: "pipe", + }); + let address; + const serverErrors = new Response(server.stderr).text(); + try { + const deadline = Date.now() + 35_000; + while (true) { + const ready = await readFile(join(data, "server.json"), "utf8").catch((error) => { + if (error.code !== "ENOENT") throw error; + }); + if (ready) { + address = JSON.parse(ready).url.replace(/^ws:/, "http:").replace(/\/ws$/, ""); + break; + } + if (server.exitCode !== null) throw new Error(`Server exited: ${await serverErrors}`); + if (Date.now() >= deadline) throw new Error("Server did not publish readiness"); + await Bun.sleep(20); + } + await check(address, token); + } finally { + try { + if (address) { + // The server can close the connection before writing the shutdown reply. + const response = await fetch(`${address}/shutdown`, { + headers: { Authorization: `Bearer ${token}` }, + method: "POST", + signal: AbortSignal.timeout(5000), + }).catch((error) => { + if (error.code !== "ECONNRESET" && error.code !== "ConnectionClosed") throw error; + }); + if (response) expect(response.ok).toBe(true); + expect(await Promise.race([server.exited, Bun.sleep(5000).then(() => null)])).toBe(0); + expect(await Bun.file(join(data, "server.json")).exists()).toBe(false); + } + } finally { + if (server.exitCode === null) server.kill("SIGKILL"); + await server.exited; + } + } +} + +test("a real local server loads and removes an indexed plugin on restart", async () => { + await withPlugin(async (plugin, root) => { + function run(args, env) { + const result = cli(root, ["plugin", ...args, "--json"], env); + expect(result.code, result.stderr).toBe(0); + return JSON.parse(result.stdout); + } + run(["build", plugin]); + await withServer(root, () => { + expect(run(["list"])).toEqual({ ok: true, plugins: [] }); + run(["add", plugin]); + expect(run(["list"])).toEqual({ ok: true, plugins: [] }); + }); + await withServer(root, (address, token) => { + const loaded = { ok: true, plugins: [{ id: "outside-fixture", version: "0.1.0", fromIndex: true }] }; + expect(run(["list"])).toEqual(loaded); + expect(run(["list"], { TOKAMAK_SERVER: address, TOKAMAK_TOKEN: token })).toEqual(loaded); + expect( + run(["list", "--server", address], { TOKAMAK_SERVER: "http://127.0.0.1:1", TOKAMAK_TOKEN: token }), + ).toEqual(loaded); + run(["remove", "outside-fixture"]); + expect(run(["list"])).toEqual(loaded); + }); + await withServer(root, () => { + expect(run(["list"])).toEqual({ ok: true, plugins: [] }); + }); + }); +}, 120_000); diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json new file mode 100644 index 00000000..05fbe9e8 --- /dev/null +++ b/packages/cli/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../build/tsconfig.json", + "include": ["src/**/*.ts"] +} diff --git a/packages/server/src/backend.rs b/packages/server/src/backend.rs index d62ea0b4..d44b53ee 100644 --- a/packages/server/src/backend.rs +++ b/packages/server/src/backend.rs @@ -24,12 +24,21 @@ impl Backend { storage: Storage, javascript: PoolConfig, root: &std::path::Path, + ) -> Result { + Self::start_plugins_with_index(storage, javascript, root, None).await + } + + pub async fn start_plugins_with_index( + storage: Storage, + javascript: PoolConfig, + root: &std::path::Path, + index: Option<&std::path::Path>, ) -> Result { let pool = Arc::new(Pool::new(javascript).map_err(|e| Error::Invalid(e.to_string()))?); let database = Database::open(storage) .await .map_err(|e| Error::Invalid(e.to_string()))?; - let server = Server::load_plugins(database, pool, root).await?; + let server = Server::load_plugins_with_index(database, pool, root, index).await?; Ok(Self { server: Arc::new(server), }) diff --git a/packages/server/src/main.rs b/packages/server/src/main.rs index c3c08116..a074b0e6 100644 --- a/packages/server/src/main.rs +++ b/packages/server/src/main.rs @@ -232,9 +232,17 @@ async fn start_backend( javascript: PoolConfig, plugins: Option, source: Option, + local: Option<&Path>, ) -> Result<(Backend, String), Box> { let result = if let Some(root) = plugins { - let backend = Backend::start_plugins(Storage::Disk(data), javascript, &root).await?; + let index = local.map(|directory| directory.join("plugin-index.json")); + let backend = Backend::start_plugins_with_index( + Storage::Disk(data), + javascript, + &root, + index.as_deref(), + ) + .await?; let hash = backend.server().plugins().await?["revision"] .as_str() .ok_or("Missing catalog revision")? @@ -346,7 +354,8 @@ async fn main() -> Result<(), Box> { let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; let plugin_root = plugins.as_ref().map(std::fs::canonicalize).transpose()?; let build_id = build_id()?; - let (backend, bundle_hash) = start_backend(data, javascript, plugins, source).await?; + let (backend, bundle_hash) = + start_backend(data, javascript, plugins, source, local.as_deref()).await?; let identity = serde_json::json!({ "url": format!("ws://{}/ws", listener.local_addr()?), "pid": std::process::id(), diff --git a/packages/server/src/plugins/catalog.rs b/packages/server/src/plugins/catalog.rs index 238f311a..ccda4a80 100644 --- a/packages/server/src/plugins/catalog.rs +++ b/packages/server/src/plugins/catalog.rs @@ -12,6 +12,7 @@ impl Catalog { previous .map(|p| p.directory.as_path()) .unwrap_or(root.parent().unwrap_or(root)), + None, ) } @@ -19,6 +20,7 @@ impl Catalog { root: &Path, previous: Option<&Self>, directory: &Path, + index_path: Option<&Path>, ) -> Result { let directories = if let Some(directory) = artifact_directory(root) { vec![directory] @@ -47,24 +49,54 @@ impl Catalog { result }; - let mut plugins = BTreeMap::new(); - + // Keep index membership fixed for this server lifetime, even across a + // reload through the server's existing plugin controls. + let index = match previous { + Some(catalog) => Arc::clone(&catalog.index), + None => Arc::new(read_index(index_path)?), + }; + let mut sources = BTreeMap::new(); for directory in directories { - // Read identity first to find reusable verified snapshots. The full - // read still verifies changed files, detecting incomplete builds. - let identity = Manifest::read(&directory)?; + let id = Manifest::read(&directory)?.plugin.id; + if sources.insert(id.clone(), directory).is_some() { + return Err(invalid(format!( + "Duplicate installed plugin identity: {id}" + ))); + } + } - let old = previous.and_then(|p| p.plugins.get(&identity.plugin.id)); + let mut indexed = BTreeSet::new(); + for (id, folder) in index.iter() { + let dist = folder.join("dist"); + if !dist.join("manifest.json").is_file() { + eprintln!( + "Skipping indexed plugin {id}: no build at {}", + dist.display() + ); + continue; + } + let actual = Manifest::read(&dist)?.plugin.id; + if actual != *id { + return Err(invalid(format!( + "Indexed plugin {id} has plugin identity {actual} in {}", + dist.display() + ))); + } + sources.insert(id.clone(), dist); + indexed.insert(id.clone()); + } + let mut plugins = BTreeMap::new(); + for (id, directory) in sources { + let old = previous.and_then(|p| p.plugins.get(&id)); let artifact = Artifact::read_reusing(&directory, old.map(AsRef::as_ref))?; - - let id = artifact.manifest.plugin.id.clone(); - - if plugins.insert(id.clone(), Arc::new(artifact)).is_some() { + if artifact.manifest.plugin.id != id { return Err(invalid(format!( - "Duplicate installed plugin identity: {id}" + "Plugin identity changed while reading {}", + directory.display() ))); } + plugins.insert(id, Arc::new(artifact)); } let order = resolve(&plugins)?; @@ -95,6 +127,8 @@ impl Catalog { plugins, revision: format!("{:x}", hash.finalize()), providers, + index, + indexed, environment, directory: directory.to_owned(), }) @@ -104,6 +138,7 @@ impl Catalog { json!({"revision":self.revision,"plugins":self.order.iter().map(|id| { let manifest = &self.plugins[id].manifest; let mut value = serde_json::to_value(manifest).expect("manifest serialization"); + value["fromIndex"] = json!(self.indexed.contains(id)); value["revision"] = json!(self.revision); value["uiUrl"] = manifest.entries.ui.as_ref().map(|ui| format!("/plugins/{id}/{}/assets/{}", self.revision, ui.code)).into(); value["stylesheetUrl"] = manifest.entries.ui.as_ref().and_then(|ui| ui.css.as_ref()).map(|css| format!("/plugins/{id}/{}/assets/{css}", self.revision)).into(); @@ -161,3 +196,29 @@ fn artifact_directory(root: &Path) -> Option { None } } + +fn read_index(path: Option<&Path>) -> Result, Error> { + let Some(path) = path else { + return Ok(BTreeMap::new()); + }; + let source = match std::fs::read(path) { + Ok(source) => source, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(BTreeMap::new()), + Err(error) => { + return Err(invalid(format!( + "Cannot read plugin index {}: {error}", + path.display() + ))); + } + }; + let entries: BTreeMap = serde_json::from_slice(&source) + .map_err(|error| invalid(format!("Invalid plugin index {}: {error}", path.display())))?; + for (id, folder) in &entries { + if !folder.is_absolute() { + return Err(invalid(format!( + "Indexed plugin {id} needs an absolute source folder" + ))); + } + } + Ok(entries) +} diff --git a/packages/server/src/plugins/mod.rs b/packages/server/src/plugins/mod.rs index a03cbca9..6620de17 100644 --- a/packages/server/src/plugins/mod.rs +++ b/packages/server/src/plugins/mod.rs @@ -26,6 +26,8 @@ pub struct Catalog { pub plugins: BTreeMap>, pub revision: String, providers: BTreeMap>, + index: Arc>, + indexed: BTreeSet, pub(crate) environment: PathBuf, pub(crate) directory: PathBuf, } diff --git a/packages/server/src/service/startup.rs b/packages/server/src/service/startup.rs index 99ebab33..d281cf19 100644 --- a/packages/server/src/service/startup.rs +++ b/packages/server/src/service/startup.rs @@ -79,14 +79,24 @@ impl Server { /// All static manifests resolve before any plugin module is evaluated. pub async fn load_plugins( + database: Database, + pool: Arc, + root: &std::path::Path, + ) -> Result { + Self::load_plugins_with_index(database, pool, root, None).await + } + + pub(crate) async fn load_plugins_with_index( mut database: Database, pool: Arc, root: &std::path::Path, + index: Option<&std::path::Path>, ) -> Result { let directory = database .directory() .unwrap_or(root.parent().unwrap_or(root)); - let loaded = match crate::plugins::Catalog::read_in_directory(root, None, directory) { + let loaded = match crate::plugins::Catalog::read_in_directory(root, None, directory, index) + { Ok(catalog) => crate::plugins::Loaded::load(catalog, &pool).await, Err(error) => Err(error), }; diff --git a/packages/server/tests/plugins.rs b/packages/server/tests/plugins.rs index 2fabac03..dea98747 100644 --- a/packages/server/tests/plugins.rs +++ b/packages/server/tests/plugins.rs @@ -863,3 +863,110 @@ async fn assert_replayed_plugin_writes( "separate mutations reused a seed" ); } + +#[tokio::test] +async fn local_server_skips_missing_and_unbuilt_index_entries_and_prefers_the_index() { + let installed = Installation::new(); + let temporary = tempfile::tempdir().unwrap(); + let local = temporary.path(); + let indexed = local.join("indexed"); + let dist = indexed.join("dist"); + std::fs::create_dir_all(&dist).unwrap(); + for file in ["manifest.json", "bundle.js", "bundle.js.map", "api.ts"] { + std::fs::copy(installed.0.join("a").join(file), dist.join(file)).unwrap(); + } + let manifest = dist.join("manifest.json"); + let mut value: Value = serde_json::from_slice(&std::fs::read(&manifest).unwrap()).unwrap(); + value["plugin"]["version"] = json!("1.0.1"); + std::fs::write(manifest, value.to_string()).unwrap(); + let unbuilt = local.join("unbuilt"); + std::fs::create_dir(&unbuilt).unwrap(); + std::fs::write( + local.join("plugin-index.json"), + json!({ "a": indexed, "missing": local.join("missing"), "unbuilt": unbuilt }).to_string(), + ) + .unwrap(); + let token = local.join("server.token"); + std::fs::write(&token, "b".repeat(64)).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&token, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + let mut process = Process( + std::process::Command::new(env!("CARGO_BIN_EXE_tokamak-server")) + .arg("--local-dir") + .arg(local) + .arg("--plugins") + .arg(&installed.0) + .env_remove("TOKAMAK_WEB_DIR") + .spawn() + .unwrap(), + ); + let ready = local.join("server.json"); + let started = std::time::Instant::now(); + while !ready.exists() { + assert!( + process.0.try_wait().unwrap().is_none(), + "server exited before readiness" + ); + assert!( + started.elapsed() < Duration::from_secs(20), + "startup timed out" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + let identity: Value = serde_json::from_slice(&std::fs::read(ready).unwrap()).unwrap(); + let base = identity["url"] + .as_str() + .unwrap() + .replace("ws://", "http://"); + let base = base.trim_end_matches("/ws"); + let http = reqwest::Client::new(); + let body = http + .get(format!("{base}/plugins")) + .bearer_auth("b".repeat(64)) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .bytes() + .await + .unwrap(); + let catalog: Value = serde_json::from_slice(&body).unwrap(); + let plugins = catalog["plugins"].as_array().unwrap(); + assert_eq!(plugins.len(), 2); + let indexed = plugins + .iter() + .find(|plugin| plugin["plugin"]["id"] == "a") + .unwrap(); + let installed = plugins + .iter() + .find(|plugin| plugin["plugin"]["id"] == "b") + .unwrap(); + assert_eq!(indexed["plugin"]["version"], "1.0.1"); + assert_eq!(indexed["fromIndex"], true); + assert_eq!(installed["fromIndex"], false); + // The server can close the connection before writing the shutdown reply. + if let Ok(response) = http + .post(format!("{base}/shutdown")) + .bearer_auth("b".repeat(64)) + .send() + .await + { + response.error_for_status().unwrap(); + } + let status = tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(status) = process.0.try_wait().unwrap() { + break status; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap(); + assert!(status.success(), "server shutdown failed: {status}"); + assert!(!local.join("server.json").exists()); +} From eba60355b7dfaccd2953b01baffd63045d020e64 Mon Sep 17 00:00:00 2001 From: JustKira Date: Fri, 20 Nov 2026 13:58:02 +0200 Subject: [PATCH 2/8] Separate indexed source selection from catalog loading --- packages/server/src/plugins/catalog.rs | 49 ++++++++++++-------- packages/server/tests/plugins.rs | 63 ++++++++++++++------------ 2 files changed, 63 insertions(+), 49 deletions(-) diff --git a/packages/server/src/plugins/catalog.rs b/packages/server/src/plugins/catalog.rs index ccda4a80..11556349 100644 --- a/packages/server/src/plugins/catalog.rs +++ b/packages/server/src/plugins/catalog.rs @@ -65,26 +65,7 @@ impl Catalog { } } - let mut indexed = BTreeSet::new(); - for (id, folder) in index.iter() { - let dist = folder.join("dist"); - if !dist.join("manifest.json").is_file() { - eprintln!( - "Skipping indexed plugin {id}: no build at {}", - dist.display() - ); - continue; - } - let actual = Manifest::read(&dist)?.plugin.id; - if actual != *id { - return Err(invalid(format!( - "Indexed plugin {id} has plugin identity {actual} in {}", - dist.display() - ))); - } - sources.insert(id.clone(), dist); - indexed.insert(id.clone()); - } + let indexed = select_indexed_sources(&index, &mut sources)?; let mut plugins = BTreeMap::new(); for (id, directory) in sources { @@ -222,3 +203,31 @@ fn read_index(path: Option<&Path>) -> Result, Error> { } Ok(entries) } + +fn select_indexed_sources( + index: &BTreeMap, + sources: &mut BTreeMap, +) -> Result, Error> { + let mut indexed = BTreeSet::new(); + for (id, folder) in index { + let dist = folder.join("dist"); + if !dist.join("manifest.json").is_file() { + eprintln!( + "Skipping indexed plugin {id}: no build at {}", + dist.display() + ); + continue; + } + let actual = Manifest::read(&dist)?.plugin.id; + if actual != *id { + return Err(invalid(format!( + "Indexed plugin {id} has plugin identity {actual} in {}", + dist.display() + ))); + } + sources.insert(id.clone(), dist); + indexed.insert(id.clone()); + } + + Ok(indexed) +} diff --git a/packages/server/tests/plugins.rs b/packages/server/tests/plugins.rs index dea98747..06d1444e 100644 --- a/packages/server/tests/plugins.rs +++ b/packages/server/tests/plugins.rs @@ -886,36 +886,8 @@ async fn local_server_skips_missing_and_unbuilt_index_entries_and_prefers_the_in json!({ "a": indexed, "missing": local.join("missing"), "unbuilt": unbuilt }).to_string(), ) .unwrap(); - let token = local.join("server.token"); - std::fs::write(&token, "b".repeat(64)).unwrap(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&token, std::fs::Permissions::from_mode(0o600)).unwrap(); - } - let mut process = Process( - std::process::Command::new(env!("CARGO_BIN_EXE_tokamak-server")) - .arg("--local-dir") - .arg(local) - .arg("--plugins") - .arg(&installed.0) - .env_remove("TOKAMAK_WEB_DIR") - .spawn() - .unwrap(), - ); + let mut process = start_index_test_server(local, &installed.0).await; let ready = local.join("server.json"); - let started = std::time::Instant::now(); - while !ready.exists() { - assert!( - process.0.try_wait().unwrap().is_none(), - "server exited before readiness" - ); - assert!( - started.elapsed() < Duration::from_secs(20), - "startup timed out" - ); - tokio::time::sleep(Duration::from_millis(20)).await; - } let identity: Value = serde_json::from_slice(&std::fs::read(ready).unwrap()).unwrap(); let base = identity["url"] .as_str() @@ -970,3 +942,36 @@ async fn local_server_skips_missing_and_unbuilt_index_entries_and_prefers_the_in assert!(status.success(), "server shutdown failed: {status}"); assert!(!local.join("server.json").exists()); } + +async fn start_index_test_server(local: &std::path::Path, installed: &std::path::Path) -> Process { + let token = local.join("server.token"); + std::fs::write(&token, "b".repeat(64)).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&token, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + let mut process = Process( + std::process::Command::new(env!("CARGO_BIN_EXE_tokamak-server")) + .arg("--local-dir") + .arg(local) + .arg("--plugins") + .arg(installed) + .env_remove("TOKAMAK_WEB_DIR") + .spawn() + .unwrap(), + ); + let started = std::time::Instant::now(); + while !local.join("server.json").exists() { + assert!( + process.0.try_wait().unwrap().is_none(), + "server exited before readiness" + ); + assert!( + started.elapsed() < Duration::from_secs(20), + "startup timed out" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + process +} From a97d773d88c9002deea063cdbe2429552b7e1d20 Mon Sep 17 00:00:00 2001 From: JustKira Date: Fri, 20 Nov 2026 13:58:38 +0200 Subject: [PATCH 3/8] Write CLI results to their designated streams --- packages/cli/src/main.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 216b8cc2..06153cc5 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -10,10 +10,10 @@ let next = "tokamak --help"; function output(data: object, human: string) { if (json) { - console.log(JSON.stringify(data)); - console.error(human); + process.stdout.write(JSON.stringify(data) + "\n"); + process.stderr.write(human + "\n"); } else { - console.log(human); + process.stdout.write(human + "\n"); } } From 29b7af23e1b6685ace610e274779a8e955e41996 Mon Sep 17 00:00:00 2001 From: JustKira Date: Fri, 20 Nov 2026 14:07:28 +0200 Subject: [PATCH 4/8] Let a CLI command declare its own flags The shared parser rejected every flag except --json, --server and --help, so a command could not take --errors, --out or --new. A command now lists its flags, the parser accepts only those, and help shows them. --- packages/cli/README.md | 10 ++++++ packages/cli/src/command.ts | 18 ++++++++++- packages/cli/src/main.ts | 55 ++++++++++++++++++++++++--------- packages/cli/tests/cli.test.mjs | 12 +++++++ 4 files changed, 80 insertions(+), 15 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index 2f76d852..5faf1509 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -55,6 +55,16 @@ To add a command, create a file under `src/commands/` that satisfies `Command`, then register it in `src/commands.ts`. The entry point handles flags, help, output and exit codes. The CLI test walks the registry to check that contract. +A command that needs its own flags declares them in `flags`, for example +`flags: { errors: { summary: "Show only errors." }, out: { value: "path", summary: "Where to write the file." } }`. +A flag with `value` takes one value; a flag without it is a switch; `short` adds a +one-letter form. The command receives them as `options.flags`. Help lists them, and a +flag the command did not declare is an error. `--json`, `--server` and `--help` work on +every command and are never declared. + +Without `--json`, a command's text goes to stdout. With `--json`, stdout holds exactly +one JSON value and the text goes to stderr. + Run the CLI tests with the real server: ```sh diff --git a/packages/cli/src/command.ts b/packages/cli/src/command.ts index 1e43140c..91507976 100644 --- a/packages/cli/src/command.ts +++ b/packages/cli/src/command.ts @@ -1,9 +1,25 @@ +/** + * A flag that belongs to one command, written `--name` on the command line. + * `value` is the placeholder shown in help (`--out `); a flag without it + * is a switch. `short` adds a one-letter form (`-t`). + */ +export interface Flag { + summary: string; + value?: string; + short?: string; +} + +/** The flags a command received: a switch is `true`, a flag with a value holds that value. */ +export type Flags = Record; + export interface Command { path: string; summary: string; example: string; args?: { name: string; optional?: boolean }; - run(args: string[], options: { server?: string }): Promise<{ data: object; text: string }>; + /** Flags only this command accepts. `--json`, `--server` and `--help` work on every command. */ + flags?: Record; + run(args: string[], options: { server?: string; flags: Flags }): Promise<{ data: object; text: string }>; } export class CommandError extends Error { diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 06153cc5..22b90f44 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -1,4 +1,4 @@ -import { CommandError } from "./command"; +import { CommandError, type Flags } from "./command"; import { commands } from "./commands"; const argv = process.argv.slice(2); @@ -6,6 +6,7 @@ const json = argv.includes("--json"); const help = argv.includes("--help"); const positional: string[] = []; let server: string | undefined; +const flags: Flags = {}; let next = "tokamak --help"; function output(data: object, human: string) { @@ -18,21 +19,43 @@ function output(data: object, human: string) { } try { - for (const [i, arg] of argv.entries()) { - if (arg === "--server") { - if (server || !argv[i + 1] || argv[i + 1]!.startsWith("-")) - throw new CommandError("Expected one address after --server", next); - server = argv[i + 1]; - } else if (argv[i - 1] === "--server" || arg === "--help" || arg === "--json") { + // Pass 1 finds the command from the leading words, because which flags are + // allowed depends on the command. Values of `--server` are skipped; a value of + // a command flag can only follow the command's own words, so it cannot change + // which command matches. + const words = argv.filter((arg, i) => !arg.startsWith("-") && argv[i - 1] !== "--server"); + const command = commands + .filter(({ path }) => path.split(" ").every((word, i) => words[i] === word)) + .sort((a, b) => b.path.length - a.path.length)[0]; + if (command) next = `tokamak ${command.path} --help`; + const declared = Object.entries(command?.flags ?? {}); + + // Pass 2 reads every token against the flags that command declares. + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]!; + if (arg === "--help" || arg === "--json") continue; + if (!arg.startsWith("-")) { + positional.push(arg); continue; - } else if (arg.startsWith("-")) { - throw new CommandError(`Unknown flag: ${arg}`, next); + } + const found = declared.find( + ([name, flag]) => arg === `--${name}` || (flag.short && arg === `-${flag.short}`), + ); + const name = arg === "--server" ? "server" : found?.[0]; + if (!name) throw new CommandError(`Unknown flag: ${arg}`, next); + if (name in flags || (name === "server" && server)) + throw new CommandError(`${arg} was given twice`, next); + if (name === "server" || found?.[1].value) { + const value = argv[++i]; + const placeholder = name === "server" ? "address" : found![1].value; + if (!value || value.startsWith("-")) + throw new CommandError(`Expected <${placeholder}> after ${arg}`, next); + if (name === "server") server = value; + else flags[name] = value; } else { - positional.push(arg); + flags[name] = true; } } - const command = commands.find(({ path }) => path.split(" ").every((word, i) => positional[i] === word)); - if (command) next = `tokamak ${command.path} --help`; const args = command ? positional.slice(command.path.split(" ").length) : []; const group = positional.join(" "); const children = commands.filter(({ path }) => !group || path.startsWith(group + " ")); @@ -42,8 +65,12 @@ try { if (help && (command || children.length)) { const text = command ? [ - `Usage: tokamak ${command.path}${command.args ? (command.args.optional ? ` [${command.args.name}]` : ` <${command.args.name}>`) : ""} [--json] [--server
] [--help]`, + `Usage: tokamak ${command.path}${command.args ? (command.args.optional ? ` [${command.args.name}]` : ` <${command.args.name}>`) : ""}${declared.map(([name, flag]) => ` [--${name}${flag.value ? ` <${flag.value}>` : ""}]`).join("")} [--json] [--server
] [--help]`, command.summary, + ...declared.map( + ([name, flag]) => + ` ${flag.short ? `-${flag.short}, ` : ""}--${name}${flag.value ? ` <${flag.value}>` : ""} ${flag.summary}`, + ), `Example: ${command.example}`, ].join("\n") : [ @@ -57,7 +84,7 @@ try { throw new CommandError(group ? `Unknown or incomplete command: ${group}` : "Missing command", next); if (command.args && !command.args.optional && !args.length) throw new CommandError(`Missing argument: ${command.args.name}`, next); - const { data, text } = await command.run(args, { server }); + const { data, text } = await command.run(args, { server, flags }); output({ ok: true, ...data }, text); } } catch (error) { diff --git a/packages/cli/tests/cli.test.mjs b/packages/cli/tests/cli.test.mjs index 88dd2b06..70150688 100644 --- a/packages/cli/tests/cli.test.mjs +++ b/packages/cli/tests/cli.test.mjs @@ -110,6 +110,18 @@ test("every registered command shares help and JSON output rules", async () => { error: { message: expect.stringContaining("Unknown flag"), next: expect.stringContaining("--help") }, }); } + // Flags a command declares must show in its help, and a flag that takes a + // value must reject a missing one. This covers commands added later. + for (const command of commands) { + for (const [name, flag] of Object.entries(command.flags ?? {})) { + const words = command.path.split(" "); + expect(cli(root, [...words, "--help"]).stdout).toContain(`--${name}`); + if (!flag.value) continue; + const missing = cli(root, [...words, `--${name}`, "--json"]); + expect(missing.code).not.toBe(0); + expect(JSON.parse(missing.stdout).error.message).toContain(`after --${name}`); + } + } for (const command of commands.filter(({ args }) => args && !args.optional)) { const missing = cli(root, [...command.path.split(" "), "--json"]); expect(missing.code).not.toBe(0); From ce0dbc7e7890562741e172a5969d5b5debffc53f Mon Sep 17 00:00:00 2001 From: JustKira Date: Fri, 20 Nov 2026 14:25:08 +0200 Subject: [PATCH 5/8] Protect CLI output, local discovery, and index updates --- packages/cli/src/command.ts | 4 +- packages/cli/src/commands/add.ts | 20 ++--- packages/cli/src/commands/list.ts | 1 + packages/cli/src/commands/remove.ts | 9 +- packages/cli/src/main.ts | 24 +++-- packages/cli/src/plugin-index.ts | 63 +++++++++++-- packages/cli/src/process.ts | 9 ++ packages/cli/src/server.ts | 80 ++++++++++++----- packages/cli/tests/cli.test.mjs | 131 ++++++++++++++++++++++++++-- 9 files changed, 281 insertions(+), 60 deletions(-) create mode 100644 packages/cli/src/process.ts diff --git a/packages/cli/src/command.ts b/packages/cli/src/command.ts index 91507976..9c711028 100644 --- a/packages/cli/src/command.ts +++ b/packages/cli/src/command.ts @@ -17,7 +17,9 @@ export interface Command { summary: string; example: string; args?: { name: string; optional?: boolean }; - /** Flags only this command accepts. `--json`, `--server` and `--help` work on every command. */ + /** Accept the shared `--server` option on this command. */ + usesServer?: boolean; + /** Flags only this command accepts. `--json` and `--help` work on every command. */ flags?: Record; run(args: string[], options: { server?: string; flags: Flags }): Promise<{ data: object; text: string }>; } diff --git a/packages/cli/src/commands/add.ts b/packages/cli/src/commands/add.ts index c961c666..7ec09dc4 100644 --- a/packages/cli/src/commands/add.ts +++ b/packages/cli/src/commands/add.ts @@ -1,7 +1,7 @@ import { realpath, stat } from "node:fs/promises"; import { readSourcePlugin } from "../../../build/src/descriptor"; import { CommandError, type Command } from "../command"; -import { readIndex, writeIndex } from "../plugin-index"; +import { updateIndex } from "../plugin-index"; export default { path: "plugin add", @@ -12,17 +12,17 @@ export default { const folder = await realpath(source!); if (!(await stat(folder)).isDirectory()) throw new Error(`Not a source folder: ${folder}`); const { id } = await readSourcePlugin(folder); - const entries = await readIndex(); - const existing = entries.get(id); - if (existing !== undefined && existing !== folder) - throw new CommandError(`Plugin ${id} is already indexed at ${existing}`, `tokamak plugin remove ${id}`); - if (!existing) { + await updateIndex((entries) => { + const existing = entries.get(id); + if (existing !== undefined && existing !== folder) + throw new CommandError(`Plugin ${id} is already indexed at ${existing}`, `tokamak plugin remove ${id}`); + if (existing) return false; entries.set(id, folder); - await writeIndex(entries); - } + return true; + }); return { - data: { id, folder, restartRequired: true }, - text: `Indexed ${id} at ${folder}. The local server picks this up at its next start.`, + data: { id, folder, restartRequired: true, precedence: "index" }, + text: `Indexed ${id} at ${folder}. The local server picks this up at its next start. The indexed build takes precedence over any ${id} from --plugins.`, }; }, } satisfies Command; diff --git a/packages/cli/src/commands/list.ts b/packages/cli/src/commands/list.ts index ed6c1c30..49a36aaa 100644 --- a/packages/cli/src/commands/list.ts +++ b/packages/cli/src/commands/list.ts @@ -5,6 +5,7 @@ type LoadedPlugin = { plugin: { id: string; version: string }; fromIndex?: boole export default { path: "plugin list", + usesServer: true, summary: "List plugins loaded by the running server.", example: "tokamak plugin list --json", async run(_, { server }) { diff --git a/packages/cli/src/commands/remove.ts b/packages/cli/src/commands/remove.ts index ff285181..eaef9764 100644 --- a/packages/cli/src/commands/remove.ts +++ b/packages/cli/src/commands/remove.ts @@ -1,5 +1,5 @@ import type { Command } from "../command"; -import { readIndex, writeIndex } from "../plugin-index"; +import { updateIndex } from "../plugin-index"; export default { path: "plugin remove", @@ -7,9 +7,10 @@ export default { args: { name: "id" }, example: "tokamak plugin remove my-plugin", async run([id]) { - const entries = await readIndex(); - if (!entries.delete(id!)) throw new Error(`Unknown indexed plugin: ${id}`); - await writeIndex(entries); + await updateIndex((entries) => { + if (!entries.delete(id!)) throw new Error(`Unknown indexed plugin: ${id}`); + return true; + }); return { data: { id, restartRequired: true }, text: `Removed ${id} from the index. The local server picks this up at its next start.`, diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 22b90f44..72c8a4b6 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -9,13 +9,15 @@ let server: string | undefined; const flags: Flags = {}; let next = "tokamak --help"; -function output(data: object, human: string) { - if (json) { - process.stdout.write(JSON.stringify(data) + "\n"); - process.stderr.write(human + "\n"); - } else { - process.stdout.write(human + "\n"); - } +// A downstream reader may close its pipe before the command writes a result. +for (const stream of [process.stdout, process.stderr]) { + stream.on("error", (error: NodeJS.ErrnoException) => process.exit(error.code === "EPIPE" ? 0 : 1)); +} + +function output(data: object, human: string, failed = false) { + if (json || failed) process.stderr.write(human + "\n"); + if (json) process.stdout.write(JSON.stringify(data) + "\n"); + else if (!failed) process.stdout.write(human + "\n"); } try { @@ -34,6 +36,7 @@ try { for (let i = 0; i < argv.length; i++) { const arg = argv[i]!; if (arg === "--help" || arg === "--json") continue; + if (!arg) throw new CommandError("Empty positional argument", next); if (!arg.startsWith("-")) { positional.push(arg); continue; @@ -41,6 +44,8 @@ try { const found = declared.find( ([name, flag]) => arg === `--${name}` || (flag.short && arg === `-${flag.short}`), ); + if (arg === "--server" && !command?.usesServer) + throw new CommandError("--server is only available on commands that use a server", next); const name = arg === "--server" ? "server" : found?.[0]; if (!name) throw new CommandError(`Unknown flag: ${arg}`, next); if (name in flags || (name === "server" && server)) @@ -65,7 +70,7 @@ try { if (help && (command || children.length)) { const text = command ? [ - `Usage: tokamak ${command.path}${command.args ? (command.args.optional ? ` [${command.args.name}]` : ` <${command.args.name}>`) : ""}${declared.map(([name, flag]) => ` [--${name}${flag.value ? ` <${flag.value}>` : ""}]`).join("")} [--json] [--server
] [--help]`, + `Usage: tokamak ${command.path}${command.args ? (command.args.optional ? ` [${command.args.name}]` : ` <${command.args.name}>`) : ""}${declared.map(([name, flag]) => ` [--${name}${flag.value ? ` <${flag.value}>` : ""}]`).join("")} [--json]${command.usesServer ? " [--server
]" : ""} [--help]`, command.summary, ...declared.map( ([name, flag]) => @@ -74,7 +79,7 @@ try { `Example: ${command.example}`, ].join("\n") : [ - `Usage: tokamak ${group ? group + " " : ""} [--json] [--server
] [--help]`, + `Usage: tokamak ${group ? group + " " : ""} [--json] [--help]`, ...children.map(({ path, summary }) => ` ${path} ${summary}`), `Example: ${children[0]!.example}`, ].join("\n"); @@ -94,6 +99,7 @@ try { output( { ok: false, error: { message, next, ...(diagnostics ? { diagnostics } : {}) } }, `${message}\nTry: ${next}`, + true, ); process.exitCode = 1; } diff --git a/packages/cli/src/plugin-index.ts b/packages/cli/src/plugin-index.ts index 02da1f4d..39198d52 100644 --- a/packages/cli/src/plugin-index.ts +++ b/packages/cli/src/plugin-index.ts @@ -1,12 +1,13 @@ import { randomUUID } from "node:crypto"; -import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, readdir, rename, rm, rmdir, writeFile } from "node:fs/promises"; import { isAbsolute, join } from "node:path"; +import { processIsAlive } from "./process"; import { dataFolder } from "./data"; const name = "plugin-index.json"; -export async function readIndex(): Promise> { - const file = join(dataFolder(), name); +async function readIndex(folder: string): Promise> { + const file = join(folder, name); const source = await readFile(file, "utf8").catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return "{}"; throw error; @@ -23,9 +24,7 @@ export async function readIndex(): Promise> { return entries; } -export async function writeIndex(entries: ReadonlyMap) { - const folder = dataFolder(); - await mkdir(folder, { recursive: true, mode: 0o700 }); +async function writeIndex(folder: string, entries: ReadonlyMap) { const scratch = join(folder, `.${name}.${randomUUID()}.tmp`); try { await writeFile(scratch, JSON.stringify(Object.fromEntries(entries), null, 2) + "\n", { mode: 0o600 }); @@ -34,3 +33,55 @@ export async function writeIndex(entries: ReadonlyMap) { await rm(scratch, { force: true }); } } + +const codeOf = (error: unknown) => (error as NodeJS.ErrnoException).code; + +async function removeEmptyLock(lock: string) { + await rmdir(lock).catch((error: unknown) => { + if (!["ENOENT", "ENOTEMPTY", "EEXIST"].includes(codeOf(error) ?? "")) throw error; + }); +} + +async function reapLock(lock: string) { + const owners = await readdir(lock).catch((error: unknown) => { + if (codeOf(error) === "ENOENT") return []; + throw error; + }); + for (const owner of owners) { + const match = /^(\d+)\./.exec(owner); + if (match && !processIsAlive(Number(match[1]))) await rm(join(lock, owner), { force: true }); + } + await removeEmptyLock(lock); +} + +export async function updateIndex(change: (entries: Map) => boolean) { + const folder = dataFolder(); + await mkdir(folder, { recursive: true, mode: 0o700 }); + const lock = join(folder, `${name}.lock`); + const owner = `${process.pid}.${randomUUID()}`; + const candidate = await mkdtemp(join(folder, `.${name}.lock-`)); + let acquired = false; + try { + // Publish a nonempty lock atomically. A reaper can unlink only the dead + // owner's uniquely named file, and cannot remove a new owner's directory. + await writeFile(join(candidate, owner), "", { flag: "wx", mode: 0o600 }); + const deadline = performance.now() + 15_000; + while (!acquired) { + try { + await rename(candidate, lock); + acquired = true; + } catch (error) { + if (!["EEXIST", "ENOTEMPTY", "EPERM", "EACCES"].includes(codeOf(error) ?? "")) throw error; + await reapLock(lock); + if (performance.now() >= deadline) throw new Error(`Timed out waiting for plugin index lock: ${lock}`); + await new Promise((done) => setTimeout(done, 25 + Math.random() * 50)); + } + } + const entries = await readIndex(folder); + if (change(entries)) await writeIndex(folder, entries); + } finally { + const directory = acquired ? lock : candidate; + await rm(join(directory, owner), { force: true }); + await removeEmptyLock(directory); + } +} diff --git a/packages/cli/src/process.ts b/packages/cli/src/process.ts new file mode 100644 index 00000000..bd013de5 --- /dev/null +++ b/packages/cli/src/process.ts @@ -0,0 +1,9 @@ +export function processIsAlive(pid: number): boolean { + if (!Number.isInteger(pid) || pid < 1 || pid > 0x7fffffff) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index d6c46616..0852a744 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -3,9 +3,15 @@ import { isIP } from "node:net"; import { join } from "node:path"; import { CommandError } from "./command"; import { dataFolder } from "./data"; +import { processIsAlive } from "./process"; function httpAddress(address: string) { - const url = new URL(address.includes("://") ? address : `http://${address}`); + let url: URL; + try { + url = new URL(address.includes("://") ? address : `http://${address}`); + } catch { + throw new Error("Invalid server address"); + } if (url.protocol === "ws:") url.protocol = "http:"; if (url.protocol === "wss:") url.protocol = "https:"; if (!["http:", "https:"].includes(url.protocol) || url.username || url.password) @@ -13,48 +19,76 @@ function httpAddress(address: string) { return url; } +function discovery(value: unknown) { + if (!value || typeof value !== "object") throw new Error("Invalid local server discovery"); + const { pid, url, bundleHash, buildId, pluginRoot } = value as Record; + if ( + typeof pid !== "number" || !processIsAlive(pid) || + typeof url !== "string" || + typeof bundleHash !== "string" || !/^[a-f0-9]{64}$/i.test(bundleHash) || + (buildId != null && typeof buildId !== "string") || + (pluginRoot != null && typeof pluginRoot !== "string") + ) throw new Error("Invalid local server identity"); + return { pid, url, bundleHash, buildId: buildId ?? null, pluginRoot: pluginRoot ?? null }; +} + async function localEndpoint() { const folder = dataFolder(); - const { url: address } = JSON.parse(await readFile(join(folder, "server.json"), "utf8")); - const ws = new URL(address); + const identity = discovery(JSON.parse(await readFile(join(folder, "server.json"), "utf8"))); + const ws = new URL(identity.url); if ( - ws.protocol !== "ws:" || - ws.pathname !== "/ws" || - ws.search || - ws.hash || + ws.protocol !== "ws:" || ws.username || ws.password || + !identity.url.endsWith("/ws") || ws.pathname !== "/ws" || ws.search || ws.hash || !(ws.hostname === "[::1]" || (isIP(ws.hostname) === 4 && ws.hostname.startsWith("127."))) - ) - throw new Error("Invalid local server discovery address"); + ) throw new Error("Invalid local server discovery address"); const file = join(folder, "server.token"); const metadata = await lstat(file); if (!metadata.isFile() || (process.platform !== "win32" && (metadata.mode & 0o077) !== 0)) throw new Error("Local server token must be a private regular file"); const token = await readFile(file, "utf8"); if (!/^[a-f0-9]{64}$/i.test(token)) throw new Error("Invalid local server token"); - return { url: httpAddress(address), token }; + const endpoint = { url: httpAddress(identity.url), token }; + const health = discovery(await get("/health", endpoint)); + if (JSON.stringify(health) !== JSON.stringify(identity)) throw new Error("Local server identity changed"); + return endpoint; } -export async function serverGet(path: string, explicit?: string): Promise { - const address = explicit ?? process.env.TOKAMAK_SERVER; - const next = address ? "tokamak --help" : "tokamak server start"; +async function get(path: string, endpoint: { url: URL; token: string }): Promise { + let response: Response; try { - const endpoint = address - ? { url: httpAddress(address), token: process.env.TOKAMAK_TOKEN } - : await localEndpoint(); - if (!endpoint.token) throw new Error("Set TOKAMAK_TOKEN for the selected server"); - const response = await fetch(new URL(path, endpoint.url), { + response = await fetch(new URL(path, endpoint.url), { headers: { Authorization: `Bearer ${endpoint.token}` }, signal: AbortSignal.timeout(5000), redirect: "error", }); - if (!response.ok) throw new Error(`Server returned HTTP ${response.status}`); + } catch { + throw new Error("Could not connect to the server"); + } + if (!response.ok) throw new Error(`Server returned HTTP ${response.status}`); + try { return await response.json(); + } catch { + throw new Error("Server returned invalid JSON"); + } +} + +export async function serverGet(path: string, explicit?: string): Promise { + const address = explicit ?? process.env.TOKAMAK_SERVER; + if (!address) { + try { + return await get(path, await localEndpoint()); + } catch { + throw new CommandError("No local server is running", "tokamak server start"); + } + } + try { + const token = process.env.TOKAMAK_TOKEN; + if (!token) throw new Error("Set TOKAMAK_TOKEN for the selected server"); + return await get(path, { url: httpAddress(address), token }); } catch (error) { - if (!address && (error as NodeJS.ErrnoException).code === "ENOENT") - throw new CommandError("No local server is running", next); throw new CommandError( - `Could not read ${path} from the ${address ? "selected server" : "local server"}: ${error instanceof Error ? error.message : String(error)}`, - next, + `Could not read ${path} from the selected server: ${error instanceof Error ? error.message : "Request failed"}`, + "tokamak --help", ); } } diff --git a/packages/cli/tests/cli.test.mjs b/packages/cli/tests/cli.test.mjs index 70150688..ed880984 100644 --- a/packages/cli/tests/cli.test.mjs +++ b/packages/cli/tests/cli.test.mjs @@ -5,8 +5,9 @@ import { randomBytes } from "node:crypto"; import { withPlugin, workspace } from "../../build/tests/outside-fixture.mjs"; function cli(root, args, extra = {}) { - const result = Bun.spawnSync([process.execPath, "run", "tokamak", ...args], { + const result = Bun.spawnSync([process.execPath, "run", "--silent", "tokamak", ...args], { cwd: workspace, + timeout: 60_000, env: { ...process.env, TOKAMAK_SERVER: undefined, @@ -30,7 +31,7 @@ test("builds an external plugin and reports its output folder", async () => { outputFolder: join(plugin, "dist"), }); }); -}, 30_000); +}, 120_000); test("returns structured type errors in both checker formats", async () => { await withPlugin(async (plugin, root) => { @@ -46,7 +47,7 @@ test("returns structured type errors in both checker formats", async () => { ); } }); -}, 30_000); +}, 120_000); test("adds and removes folders and protects an existing id", async () => { await withPlugin(async (plugin, root) => { @@ -84,7 +85,8 @@ test("tells the user how to start a missing local server", async () => { await withPlugin((_, root) => { const result = cli(root, ["plugin", "list"]); expect(result.code).not.toBe(0); - expect(result.stdout).toContain("tokamak server start"); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain("tokamak server start"); }); }); @@ -94,7 +96,12 @@ test("every registered command shares help and JSON output rules", async () => { expect(commands.map(({ path }) => path)).toEqual( expect.arrayContaining(["plugin build", "plugin add", "plugin remove", "plugin list"]), ); - for (const path of ["", "plugin", ...commands.map((command) => command.path)]) { + const paths = new Set([""]); + for (const { path } of commands) { + const words = path.split(" "); + for (let n = 1; n <= words.length; n++) paths.add(words.slice(0, n).join(" ")); + } + for (const path of paths) { const words = path ? path.split(" ") : []; const shown = cli(root, [...words, "--help"]); expect(shown.code, shown.stderr).toBe(0); @@ -131,7 +138,7 @@ test("every registered command shares help and JSON output rules", async () => { expect(unknown.code).not.toBe(0); expect(JSON.parse(unknown.stdout).error.next).toBe("tokamak --help"); }); -}, 30_000); +}, 120_000); async function withServer(root, check) { const data = join(root, "data"); @@ -157,7 +164,7 @@ async function withServer(root, check) { let address; const serverErrors = new Response(server.stderr).text(); try { - const deadline = Date.now() + 35_000; + const deadline = Date.now() + 90_000; while (true) { const ready = await readFile(join(data, "server.json"), "utf8").catch((error) => { if (error.code !== "ENOENT") throw error; @@ -220,4 +227,114 @@ test("a real local server loads and removes an indexed plugin on restart", async expect(run(["list"])).toEqual({ ok: true, plugins: [] }); }); }); +}, 330_000); + +test("rejects empty input and server selection on index commands without leaking credentials", async () => { + await withPlugin((plugin, root) => { + const empty = cli(root, ["plugin", "build", "", "--json"]); + expect(empty.code).not.toBe(0); + expect(JSON.parse(empty.stdout).error.message).toContain("Empty"); + const address = "http://user:private-password@127.0.0.1:1"; + const add = cli(root, ["plugin", "add", plugin, "--server", address, "--json"]); + expect(add.code).not.toBe(0); + expect(JSON.parse(add.stdout).error.message).toContain("--server"); + expect(cli(root, ["plugin", "add", "--help"]).stdout).not.toContain("--server"); + expect(cli(root, ["plugin", "list", "--help"]).stdout).toContain("--server"); + const list = cli(root, ["plugin", "list", "--server", address, "--json"], { TOKAMAK_TOKEN: "test" }); + expect(list.code).not.toBe(0); + expect(list.stdout + list.stderr + add.stdout + add.stderr).not.toContain("private-password"); + expect(Bun.file(join(root, "data", "plugin-index.json")).size).toBe(0); + }); +}, 120_000); + +test("closed stdout ends quietly", async () => { + await withPlugin(async (_, root) => { + const { spawn } = await import("node:child_process"); + const child = spawn(process.execPath, ["packages/cli/src/main.ts", "--help", "--json"], { + cwd: workspace, + env: { ...process.env, TOKAMAK_SERVER_DIR: join(root, "data") }, + stdio: ["ignore", "pipe", "pipe"], + timeout: 30_000, + }); + let stderr = ""; + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.stdout.destroy(); + const [code, signal] = await new Promise((done, reject) => { + child.once("error", reject); + child.once("close", (...result) => done(result)); + }); + expect(signal).toBeNull(); + expect(code).toBe(0); + expect(stderr).not.toMatch(/EPIPE|stack trace|at .*\.ts:/); + }); +}, 60_000); + +test("concurrent adds preserve every update and recover an abandoned lock", async () => { + await withPlugin(async (_, root) => { + const data = join(root, "data"); + const lock = join(data, "plugin-index.json.lock"); + await mkdir(lock, { recursive: true }); + await writeFile(join(lock, "2147483647.abandoned"), ""); + const expected = Object.fromEntries( + Array.from({ length: 12 }, (_, i) => [`concurrent-${i}`, join(root, `concurrent-${i}`)]), + ); + await Promise.all(Object.entries(expected).map(async ([id, folder]) => { + await mkdir(folder); + await writeFile(join(folder, "package.json"), JSON.stringify({ name: id })); + })); + await Promise.all(Object.values(expected).map(async (folder) => { + const child = Bun.spawn([process.execPath, "run", "--silent", "tokamak", "plugin", "add", folder, "--json"], { + cwd: workspace, + env: { ...process.env, TOKAMAK_SERVER_DIR: data }, + stdout: "pipe", + stderr: "pipe", + timeout: 60_000, + }); + const [code, stdout, stderr] = await Promise.all([ + child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(), + ]); + expect(code, stderr).toBe(0); + expect(JSON.parse(stdout).ok).toBe(true); + })); + expect(JSON.parse(await readFile(join(data, "plugin-index.json"), "utf8"))).toEqual(expected); + expect(await Bun.file(join(lock, "2147483647.abandoned")).exists()).toBe(false); + }); +}, 120_000); + +test("stale discovery sends no token and mismatched health cannot supply a catalog", async () => { + await withPlugin(async (_, root) => { + const data = join(root, "data"); + await mkdir(data); + await writeFile(join(data, "server.token"), "a".repeat(64), { mode: 0o600 }); + const requests = []; + const listener = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch(request) { + requests.push(new URL(request.url).pathname); + return Response.json({ plugins: [] }); + } }); + const address = `http://127.0.0.1:${listener.port}`; + const identity = { pid: 2147483647, url: `ws://127.0.0.1:${listener.port}/ws`, bundleHash: "b".repeat(64) }; + async function list(env = {}) { + const child = Bun.spawn([process.execPath, "run", "--silent", "tokamak", "plugin", "list", "--json"], { + cwd: workspace, + env: { ...process.env, TOKAMAK_SERVER: undefined, TOKAMAK_TOKEN: undefined, TOKAMAK_SERVER_DIR: data, ...env }, + stdout: "pipe", stderr: "pipe", timeout: 30_000, + }); + const [code, stdout] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]); + expect(code).not.toBe(0); + return JSON.parse(stdout).error; + } + try { + const discovery = join(data, "server.json"); + await writeFile(discovery, JSON.stringify(identity)); + expect((await list()).next).toBe("tokamak server start"); + expect(requests).toEqual([]); + await writeFile(discovery, JSON.stringify({ ...identity, pid: process.pid })); + expect((await list()).next).toBe("tokamak server start"); + expect(requests).toEqual(["/health"]); + expect((await list({ TOKAMAK_SERVER: address })).message).toContain("TOKAMAK_TOKEN"); + expect(requests).toEqual(["/health"]); + } finally { + listener.stop(true); + } + }); }, 120_000); From b9afe502681e85e145930844fa79591bdfef55af Mon Sep 17 00:00:00 2001 From: JustKira Date: Fri, 20 Nov 2026 14:37:05 +0200 Subject: [PATCH 6/8] Recover local server startup and report structured build errors --- packages/build/src/build.ts | 124 ++++++++------ packages/build/src/contributions.ts | 10 +- packages/build/src/diagnostics.ts | 103 ++++++++++++ packages/build/src/vite.ts | 15 +- packages/cli/README.md | 56 ++++--- packages/cli/src/commands/add.ts | 17 +- packages/cli/src/commands/build.ts | 55 ++----- packages/cli/src/main.ts | 56 +++++-- packages/cli/src/plugin-build.ts | 32 ++++ packages/cli/src/plugin-index.ts | 3 +- packages/cli/src/server.ts | 58 +++++-- packages/cli/tests/build.test.mjs | 113 +++++++++++++ packages/cli/tests/cli-fixture.mjs | 19 +++ packages/cli/tests/cli.test.mjs | 137 ++++++++-------- packages/sdk/tools/bundle.mjs | 4 - packages/server/src/plugins/catalog.rs | 217 ++++++++++++------------- packages/server/src/plugins/grants.rs | 19 +++ packages/server/src/plugins/loading.rs | 136 +++++++++++++++- packages/server/src/plugins/mod.rs | 1 - packages/server/src/service/startup.rs | 14 +- packages/server/tests/plugins.rs | 189 ++++++++++++++++++++- 21 files changed, 1011 insertions(+), 367 deletions(-) create mode 100644 packages/build/src/diagnostics.ts create mode 100644 packages/cli/src/plugin-build.ts create mode 100644 packages/cli/tests/build.test.mjs create mode 100644 packages/cli/tests/cli-fixture.mjs diff --git a/packages/build/src/build.ts b/packages/build/src/build.ts index 85da0517..76c07f7a 100644 --- a/packages/build/src/build.ts +++ b/packages/build/src/build.ts @@ -1,13 +1,14 @@ /** Public artifact build: maintained compilers, one deterministic publisher. */ -import { build } from "vite"; -import { existsSync } from "node:fs"; +import { build, resolveConfig, type InlineConfig } from "vite"; +import { existsSync, realpathSync } from "node:fs"; import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; +import { buildDiagnostics } from "./diagnostics"; import { tokamak } from "./vite"; const args = process.argv.slice(2); -const reportResult = args[0] === "--result-json"; -if (reportResult) args.shift(); +const mode = args[0] === "--result-json" || args[0] === "--check-index" ? args.shift() : undefined; +const report = (result: object) => console.log(`Tokamak build result: ${JSON.stringify(result)}`); if ( args.length > 2 || (args.length === 2 && args[0] !== "--config") || @@ -22,54 +23,75 @@ const config = : existsSync(resolve(root, "vite.config.ts")) ? resolve(root, "vite.config.ts") : false; -let directory: string | undefined; -const result = await build({ - ...(config ? { configFile: config } : { root, configFile: false }), - build: { write: false }, - plugins: [ - ...(!config ? [tokamak()] : []), - { - name: "tokamak-output", - configResolved(config) { - if (config.build.write !== false) throw new Error("Generate all output before publishing"); - directory = resolve(config.root, config.build.outDir); +async function run() { + let directory: string | undefined; + function verifyOutput(output: string) { + if (mode && output !== resolve(realpathSync(root), "dist") && output !== resolve(root, "dist")) + throw new Error(`Indexed plugin builds must use ${resolve(root, "dist")} (configured: ${output})`); + } + const options: InlineConfig = { + ...(config ? { root, configFile: config } : { root, configFile: false }), + build: { write: false }, + plugins: [ + ...(!config ? [tokamak()] : []), + { + name: "tokamak-output", + configResolved(config) { + if (config.build.write !== false) throw new Error("Generate all output before publishing"); + directory = resolve(config.root, config.build.outDir); + verifyOutput(directory); + }, }, - }, - ], -}); -if (!directory) throw new Error("Artifact output directory was not resolved"); -const files = (Array.isArray(result) ? result : [result]).flatMap((output) => { - if (!("output" in output)) throw new Error("Artifact build does not support watch mode"); - return output.output; -}); -// Manifest last; the M1 installer still needs its own complete-artifact staging. -files.sort( - (a, b) => - Number(a.fileName === "manifest.json") - Number(b.fileName === "manifest.json") || - a.fileName.localeCompare(b.fileName), -); -let changed = 0; -let unchanged = 0; -for (const file of files) { - const bytes = Buffer.from(file.type === "chunk" ? file.code : file.source); - const destination = resolve(directory, file.fileName); - try { - if (bytes.equals(await readFile(destination))) { - unchanged++; - continue; - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + ], + }; + if (mode === "--check-index") { + if (config) await resolveConfig(options, "build", "production", "production"); + report({ ok: true }); + return; } - await mkdir(dirname(destination), { recursive: true }); - const temporary = `${destination}.${process.pid}.tmp`; - try { - await writeFile(temporary, bytes); - await rename(temporary, destination); - } finally { - await rm(temporary, { force: true }); + const result = await build(options); + if (!directory) throw new Error("Artifact output directory was not resolved"); + const files = (Array.isArray(result) ? result : [result]).flatMap((output) => { + if (!("output" in output)) throw new Error("Artifact build does not support watch mode"); + return output.output; + }); + // Manifest last; the M1 installer still needs its own complete-artifact staging. + files.sort( + (a, b) => + Number(a.fileName === "manifest.json") - Number(b.fileName === "manifest.json") || + a.fileName.localeCompare(b.fileName), + ); + let changed = 0; + let unchanged = 0; + for (const file of files) { + const bytes = Buffer.from(file.type === "chunk" ? file.code : file.source); + const destination = resolve(directory, file.fileName); + try { + if (bytes.equals(await readFile(destination))) { + unchanged++; + continue; + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + await mkdir(dirname(destination), { recursive: true }); + const temporary = `${destination}.${process.pid}.tmp`; + try { + await writeFile(temporary, bytes); + await rename(temporary, destination); + } finally { + await rm(temporary, { force: true }); + } + changed++; } - changed++; + if (mode) report({ ok: true, outputFolder: directory }); + else console.log(`Plugin artifact: ${changed} changed, ${unchanged} unchanged`); +} + +try { + await run(); +} catch (error) { + if (!mode) throw error; + report({ ok: false, diagnostics: buildDiagnostics(error, root) }); + process.exitCode = 1; } -if (reportResult) console.log(`Tokamak build result: ${JSON.stringify({ outputFolder: directory })}`); -else console.log(`Plugin artifact: ${changed} changed, ${unchanged} unchanged`); diff --git a/packages/build/src/contributions.ts b/packages/build/src/contributions.ts index b8f93cc7..abbdd339 100644 --- a/packages/build/src/contributions.ts +++ b/packages/build/src/contributions.ts @@ -1,6 +1,7 @@ import { dirname } from "node:path"; import { readPluginCompilerOptions } from "../../sdk/tools/plugin-boundary.mjs"; import { clientEntry } from "./descriptor.ts"; +import { BuildError, typescriptDiagnostics } from "./diagnostics.ts"; /** Use the compiler's export/alias graph, as backend exports do. Keep TS 5.9.3: * 6.0.3 is the newest JS API release; 7.x has no JS API. Upgrading belongs to * a separate tooling change, not the plugin wire-format implementation. */ @@ -36,14 +37,7 @@ export function uiContributions(root: string) { ...program.getGlobalDiagnostics(), ...program.getSemanticDiagnostics(), ]; - if (diagnostics.length) - throw new Error( - ts.formatDiagnostics(diagnostics, { - getCanonicalFileName: (file) => file, - getCurrentDirectory: () => root, - getNewLine: () => "\n", - }), - ); + if (diagnostics.length) throw new BuildError(typescriptDiagnostics(diagnostics, root)); const checker = program.getTypeChecker(); const source = program.getSourceFile(entry); if (!source) throw new Error(`Missing plugin UI entry ${entry}`); diff --git a/packages/build/src/diagnostics.ts b/packages/build/src/diagnostics.ts new file mode 100644 index 00000000..a29d7850 --- /dev/null +++ b/packages/build/src/diagnostics.ts @@ -0,0 +1,103 @@ +import { resolve } from "node:path"; +import type ts from "typescript"; + +export interface BuildDiagnostic { + message: string; + file?: string; + line?: number; + column?: number; + code?: string; +} + +export class BuildError extends Error { + constructor(readonly diagnostics: BuildDiagnostic[]) { + super( + diagnostics + .map( + ({ file, line, column, code, message }) => + `${file ? `${file}(${line},${column}): ` : ""}${code ? `error ${code}: ` : ""}${message}`, + ) + .join("\n"), + ); + } +} + +export function typescriptDiagnostics( + diagnostics: readonly ts.Diagnostic[], + root: string, +): BuildDiagnostic[] { + function message(text: string | ts.DiagnosticMessageChain, indent = ""): string { + return typeof text === "string" + ? text + : [ + text.messageText, + ...(text.next ?? []).map((child) => indent + " " + message(child, indent + " ")), + ].join("\n"); + } + return diagnostics.map((diagnostic) => { + const result = { code: `TS${diagnostic.code}`, message: message(diagnostic.messageText) }; + if (!diagnostic.file || diagnostic.start === undefined) return result; + const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start); + return { + ...result, + file: resolve(root, diagnostic.file.fileName), + line: line + 1, + column: character + 1, + }; + }); +} + +const xmlEntities: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" }; +const unescapeXml = (value: string) => + value.replace(/&(amp|lt|gt|quot|apos);/g, (_, key) => xmlEntities[key]!); + +export function checkerDiagnostics(output: string, cwd: string): BuildDiagnostic[] { + const tagged = /]*)>([\s\S]*?)<\/error>/g; + const blocks = [...output.matchAll(tagged)]; + if (blocks.length) { + return blocks.map(([, attributes, text]) => { + const fields = Object.fromEntries( + [...attributes!.matchAll(/(\w+)="([^"]*)"/g)].map(([, key, value]) => [key, unescapeXml(value!)]), + ); + return { + ...location(cwd, fields.file, Number(fields.line), Number(fields.column)), + code: fields.code, + message: unescapeXml(text!.split(/^\r?$/m, 1)[0]!.trim()), + }; + }); + } + const plain = /^(?:(.+)\((\d+),(\d+)\): )?error (TS\d+): (.*(?:\n[ \t]+[^\n]+)*)$/gm; + return [...output.matchAll(plain)].map(([, file, line, column, code, message]) => ({ + ...location(cwd, file, Number(line), Number(column)), + code, + message: message!.trimEnd(), + })); +} + +function location(root: string, file?: string, line?: number, column?: number) { + return file && line && Number.isInteger(line) + ? { file: resolve(root, file), line, ...(column && Number.isInteger(column) ? { column } : {}) } + : {}; +} + +export function buildDiagnostics(error: unknown, root: string): BuildDiagnostic[] { + if (!(error instanceof Error)) return [{ message: String(error) }]; + const detail = error as Error & { + diagnostics?: BuildDiagnostic[]; + errors?: unknown[]; + loc?: { file?: string; line?: number; column?: number }; + id?: string; + code?: string; + }; + if (detail.diagnostics?.length) return detail.diagnostics; + if (Array.isArray(detail.errors) && detail.errors.length) + return detail.errors.flatMap((child) => buildDiagnostics(child, root)); + if (detail.cause) return buildDiagnostics(detail.cause, root); + return [ + { + ...location(root, detail.loc?.file ?? detail.id, detail.loc?.line, detail.loc?.column), + ...(typeof detail.code === "string" ? { code: detail.code } : {}), + message: detail.message, + }, + ]; +} diff --git a/packages/build/src/vite.ts b/packages/build/src/vite.ts index 76b19799..f0fd8340 100644 --- a/packages/build/src/vite.ts +++ b/packages/build/src/vite.ts @@ -11,6 +11,7 @@ import { artifactBundle } from "./artifact-bundle.ts"; import solid from "@solidjs/vite-plugin"; import { clientSurface } from "./client-surface.ts"; import { uiBuildConfig, clientAssets } from "./ui-build-config.ts"; +import { BuildError, checkerDiagnostics } from "./diagnostics.ts"; import { publicFunctions } from "./exports.ts"; import { readSourcePlugin, @@ -100,10 +101,20 @@ export function tokamak( plugin.id, ...(plugin.runtime === "worker" ? ["--worker"] : []), ], - { stdio: "inherit" }, + { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }, ); if (bundled.error) throw bundled.error; - if (bundled.status !== 0) throw new Error("SDK backend bundling failed"); + process.stdout.write(bundled.stdout); + process.stderr.write(bundled.stderr); + if (bundled.status !== 0) { + const diagnostics = checkerDiagnostics( + bundled.stdout + "\n" + bundled.stderr, + realpathSync(server), + ); + throw new BuildError( + diagnostics.length ? diagnostics : [{ message: "SDK server part bundling failed" }], + ); + } // The bundle's describe() carries the schema/index registry evaluated // in QuickJS. Keep that source authoritative instead of importing user // schema modules into the build host or duplicating them in manifests. diff --git a/packages/cli/README.md b/packages/cli/README.md index 5faf1509..eab82e66 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -3,18 +3,22 @@ Run from a checkout with the pinned Bun: ```sh -proto run bun -- run tokamak --help -proto run bun -- run tokamak plugin build /path/to/my-plugin --json -proto run bun -- run tokamak plugin add /path/to/my-plugin -proto run bun -- run tokamak plugin list --json -proto run bun -- run tokamak plugin remove my-plugin +proto run bun -- run --silent tokamak --help +proto run bun -- run --silent tokamak plugin build /path/to/my-plugin --json +proto run bun -- run --silent tokamak plugin add /path/to/my-plugin +proto run bun -- run --silent tokamak plugin list --json +proto run bun -- run --silent tokamak plugin remove my-plugin ``` `build` runs the artifact build once, defaults to the current folder, and doesn't need a server. `add` reads the plugin's static identity from `plugin.ts` (or the build's `package.json` convention), and doesn't build it. `add` and `remove` only edit the index; the local server picks up membership changes on -its next start. +its next start. Both commands serialize index updates with an exclusive owner +file: a dead owner can be removed, and a live owner is never displaced. An +update that cannot get the lock within 15 seconds fails. Indexed builds must use +`/dist`; both build and add reject a Vite configuration that changes +that location. The app data folder contains `plugin-index.json`, a JSON object mapping each plugin id to its absolute source folder, for example: @@ -25,31 +29,33 @@ plugin id to its absolute source folder, for example: At startup the local server loads each indexed folder's `dist` artifact in addition to its `--plugins` folder. A built indexed plugin wins when both -sources contain the same id. A missing index is empty; missing source folders -and entries without a build are logged and skipped. +sources contain the same id; startup logs which copy it replaces. A missing +index is empty. Unreadable or malformed indexes are logged and ignored; bad +entries are logged and skipped, leaving any copy from `--plugins` available. `list` asks the running local server for its catalog. The CLI discovers it through `server.json` and authenticates with `server.token` in the app data folder. Set `TOKAMAK_SERVER_DIR` to use a different app data folder, including for tests. For a server at a specific address, set `TOKAMAK_SERVER` and `TOKAMAK_TOKEN`, or supply `--server
` with `TOKAMAK_TOKEN`. The flag takes precedence. -The CLI never starts a server. +Local discovery checks the process and compares the authenticated health +identity to the discovery file. The CLI never starts a server. With `--json`, stdout contains one JSON value and human text goes to stderr. -Without it, human text goes to stdout. Exit status is zero on success and +Without it, normal output goes to stdout and errors go to stderr. Exit status is zero on success and nonzero on failure. The CLI never prompts. Success has `ok: true` and: -| Command | Additional result fields | -| --------------- | --------------------------------------------------- | -| `plugin build` | `id`, `outputFolder` | -| `plugin add` | `id`, `folder`, `restartRequired: true` | -| `plugin remove` | `id`, `restartRequired: true` | -| `plugin list` | `plugins`: an array of `{ id, version, fromIndex }` | -| `--help` | `help` | +| Command | Additional result fields | +| --------------- | -------------------------------------------------------------- | +| `plugin build` | `id`, `outputFolder` | +| `plugin add` | `id`, `folder`, `restartRequired: true`, `precedence: "index"` | +| `plugin remove` | `id`, `restartRequired: true` | +| `plugin list` | `plugins`: an array of `{ id, version, fromIndex }` | +| `--help` | `help` | -Failures return `{ ok: false, error: { message, next } }`. Build failures with -TypeScript locations also return `error.diagnostics`, an array of -`{ file, line, column, code, message }`. +Failures return `{ ok: false, error: { message, next } }`. Artifact build and +configuration failures include `error.diagnostics`, an array with a full +`message` and, when available, `file` (absolute), `line`, `column`, and `code`. To add a command, create a file under `src/commands/` that satisfies `Command`, then register it in `src/commands.ts`. The entry point handles flags, help, @@ -59,15 +65,13 @@ A command that needs its own flags declares them in `flags`, for example `flags: { errors: { summary: "Show only errors." }, out: { value: "path", summary: "Where to write the file." } }`. A flag with `value` takes one value; a flag without it is a switch; `short` adds a one-letter form. The command receives them as `options.flags`. Help lists them, and a -flag the command did not declare is an error. `--json`, `--server` and `--help` work on -every command and are never declared. - -Without `--json`, a command's text goes to stdout. With `--json`, stdout holds exactly -one JSON value and the text goes to stderr. +flag the command did not declare is an error. `--json` and `--help` are shared. +Set `usesServer: true` to accept the shared `--server
` option and show +it in that command's help. Run the CLI tests with the real server: ```sh cargo build -p server --bin tokamak-server -proto run bun -- test packages/cli/tests/cli.test.mjs +proto run bun -- test packages/cli/tests ``` diff --git a/packages/cli/src/commands/add.ts b/packages/cli/src/commands/add.ts index 7ec09dc4..63c5ccd9 100644 --- a/packages/cli/src/commands/add.ts +++ b/packages/cli/src/commands/add.ts @@ -1,6 +1,8 @@ -import { realpath, stat } from "node:fs/promises"; +import { access, realpath, stat } from "node:fs/promises"; +import { join } from "node:path"; import { readSourcePlugin } from "../../../build/src/descriptor"; import { CommandError, type Command } from "../command"; +import { runBuilder } from "../plugin-build"; import { updateIndex } from "../plugin-index"; export default { @@ -11,11 +13,22 @@ export default { async run([source]) { const folder = await realpath(source!); if (!(await stat(folder)).isDirectory()) throw new Error(`Not a source folder: ${folder}`); + const hasConfig = await access(join(folder, "vite.config.ts")).then( + () => true, + (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false; + throw error; + }, + ); + if (hasConfig) await runBuilder(folder, "--check-index"); const { id } = await readSourcePlugin(folder); await updateIndex((entries) => { const existing = entries.get(id); if (existing !== undefined && existing !== folder) - throw new CommandError(`Plugin ${id} is already indexed at ${existing}`, `tokamak plugin remove ${id}`); + throw new CommandError( + `Plugin ${id} is already indexed at ${existing}`, + `tokamak plugin remove ${id}`, + ); if (existing) return false; entries.set(id, folder); return true; diff --git a/packages/cli/src/commands/build.ts b/packages/cli/src/commands/build.ts index b5972801..b8356205 100644 --- a/packages/cli/src/commands/build.ts +++ b/packages/cli/src/commands/build.ts @@ -1,25 +1,7 @@ -import { execFile } from "node:child_process"; import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { promisify } from "node:util"; import { CommandError, type Command } from "../command"; - -const builder = fileURLToPath(new URL("../../../build/src/build.ts", import.meta.url)); - -function diagnostics(message: string) { - const plain = /^(.*)\((\d+),(\d+)\): error (TS\d+): (.*)$/gm; - const tagged = /\s*([^<]*)/g; - return [...message.matchAll(plain), ...message.matchAll(tagged)].map( - ([, file, line, column, code, text]) => ({ - file, - line: Number(line), - column: Number(column), - code, - message: text?.trim(), - }), - ); -} +import { runBuilder } from "../plugin-build"; export default { path: "plugin build", @@ -27,28 +9,19 @@ export default { args: { name: "folder", optional: true }, example: "tokamak plugin build . --json", async run([folder]) { - const { stdout } = await promisify(execFile)( - process.execPath, - [builder, "--result-json", resolve(folder ?? ".")], - { maxBuffer: 16 * 1024 * 1024 }, - ).catch((error: { stdout?: string; stderr?: string; message: string }) => { - const message = [error.stdout, error.stderr].filter(Boolean).join("\n").trim() || error.message; - const details = diagnostics(message); - throw new CommandError( - `Plugin build failed: ${message}`, - "tokamak plugin build --help", - details.length ? details : undefined, - ); - }); - const result = stdout - .split("\n") - .reverse() - .find((line) => line.startsWith("Tokamak build result: ")); - if (!result) throw new Error("The build did not report its output folder"); - const { outputFolder } = JSON.parse(result.slice("Tokamak build result: ".length)); - const { - plugin: { id }, - } = JSON.parse(await readFile(resolve(outputFolder, "manifest.json"), "utf8")); + const { outputFolder } = await runBuilder(resolve(folder ?? "."), "--result-json"); + if (!outputFolder) throw new Error("The build did not report its output folder"); + const id = await readFile(resolve(outputFolder, "manifest.json"), "utf8") + .then((source) => { + const id: unknown = JSON.parse(source)?.plugin?.id; + if (typeof id !== "string" || !id) throw new Error("The build manifest has no plugin id"); + return id; + }) + .catch((error: Error) => { + throw new CommandError("Plugin build failed", "tokamak plugin build --help", [ + { message: `Could not read the build manifest: ${error.message}` }, + ]); + }); return { data: { id, outputFolder }, text: `Built ${id}: ${outputFolder}` }; }, } satisfies Command; diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 72c8a4b6..0745996c 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -10,14 +10,39 @@ const flags: Flags = {}; let next = "tokamak --help"; // A downstream reader may close its pipe before the command writes a result. -for (const stream of [process.stdout, process.stderr]) { - stream.on("error", (error: NodeJS.ErrnoException) => process.exit(error.code === "EPIPE" ? 0 : 1)); +function endOnStreamError(error: unknown): never { + process.exit((error as NodeJS.ErrnoException).code === "EPIPE" ? 0 : 1); } -function output(data: object, human: string, failed = false) { - if (json || failed) process.stderr.write(human + "\n"); - if (json) process.stdout.write(JSON.stringify(data) + "\n"); - else if (!failed) process.stdout.write(human + "\n"); +for (const stream of [process.stdout, process.stderr]) stream.on("error", endOnStreamError); + +function write(stream: NodeJS.WriteStream, text: string) { + return new Promise((done) => { + try { + stream.write(text + "\n", (error) => { + if (error) endOnStreamError(error); + done(); + }); + } catch (error) { + endOnStreamError(error); + } + }); +} + +async function output(data: object, human: string, failed = false) { + if (json) await write(process.stdout, JSON.stringify(data)); + await write(json || failed ? process.stderr : process.stdout, human); +} + +function diagnosticText(diagnostics: object[]) { + return diagnostics.flatMap((entry) => { + if (!("message" in entry) || typeof entry.message !== "string") return []; + const where = + "file" in entry && typeof entry.file === "string" && "line" in entry + ? `${entry.file}:${entry.line}: ` + : ""; + return [where + entry.message]; + }); } try { @@ -47,7 +72,11 @@ try { if (arg === "--server" && !command?.usesServer) throw new CommandError("--server is only available on commands that use a server", next); const name = arg === "--server" ? "server" : found?.[0]; - if (!name) throw new CommandError(`Unknown flag: ${arg}`, next); + if (!name) + throw new CommandError( + `Unknown flag: ${arg.match(/^--?[a-zA-Z0-9-]+/)?.[0] ?? "(invalid spelling)"}`, + next, + ); if (name in flags || (name === "server" && server)) throw new CommandError(`${arg} was given twice`, next); if (name === "server" || found?.[1].value) { @@ -65,7 +94,7 @@ try { const group = positional.join(" "); const children = commands.filter(({ path }) => !group || path.startsWith(group + " ")); if (command && args.length > Number(Boolean(command.args))) { - throw new CommandError(`Unexpected argument: ${args.at(-1)}`, next); + throw new CommandError("Unexpected positional argument", next); } if (help && (command || children.length)) { const text = command @@ -83,22 +112,21 @@ try { ...children.map(({ path, summary }) => ` ${path} ${summary}`), `Example: ${children[0]!.example}`, ].join("\n"); - output({ ok: true, help: text }, text); + await output({ ok: true, help: text }, text); } else { - if (!command) - throw new CommandError(group ? `Unknown or incomplete command: ${group}` : "Missing command", next); + if (!command) throw new CommandError(group ? "Unknown or incomplete command" : "Missing command", next); if (command.args && !command.args.optional && !args.length) throw new CommandError(`Missing argument: ${command.args.name}`, next); const { data, text } = await command.run(args, { server, flags }); - output({ ok: true, ...data }, text); + await output({ ok: true, ...data }, text); } } catch (error) { const message = error instanceof Error ? error.message : String(error); if (error instanceof CommandError) next = error.next; const diagnostics = error instanceof CommandError ? error.diagnostics : undefined; - output( + await output( { ok: false, error: { message, next, ...(diagnostics ? { diagnostics } : {}) } }, - `${message}\nTry: ${next}`, + [message, ...diagnosticText(diagnostics ?? []), `Try: ${next}`].join("\n"), true, ); process.exitCode = 1; diff --git a/packages/cli/src/plugin-build.ts b/packages/cli/src/plugin-build.ts new file mode 100644 index 00000000..235fc717 --- /dev/null +++ b/packages/cli/src/plugin-build.ts @@ -0,0 +1,32 @@ +import { execFile } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { CommandError } from "./command"; +import type { BuildDiagnostic } from "../../build/src/diagnostics"; + +const builder = fileURLToPath(new URL("../../build/src/build.ts", import.meta.url)); +const prefix = "Tokamak build result: "; + +type BuildResult = { ok: true; outputFolder?: string } | { ok: false; diagnostics: BuildDiagnostic[] }; + +export async function runBuilder(folder: string, mode: "--result-json" | "--check-index") { + const processResult = promisify(execFile)(process.execPath, [builder, mode, folder], { + maxBuffer: 16 * 1024 * 1024, + }); + processResult.child.stdin?.end(); + const { stdout, failed } = await processResult.then( + ({ stdout }) => ({ stdout, failed: false }), + (error: { stdout?: string }) => ({ stdout: error.stdout ?? "", failed: true }), + ); + const record = stdout + .split("\n") + .reverse() + .find((line) => line.startsWith(prefix)); + const result: BuildResult | undefined = record ? JSON.parse(record.slice(prefix.length)) : undefined; + if (result?.ok && !failed) return result; + throw new CommandError( + mode === "--result-json" ? "Plugin build failed" : "Could not add plugin", + mode === "--result-json" ? "tokamak plugin build --help" : "tokamak plugin add --help", + result?.ok === false ? result.diagnostics : [{ message: "The artifact builder did not return a result" }], + ); +} diff --git a/packages/cli/src/plugin-index.ts b/packages/cli/src/plugin-index.ts index 39198d52..9619607f 100644 --- a/packages/cli/src/plugin-index.ts +++ b/packages/cli/src/plugin-index.ts @@ -73,7 +73,8 @@ export async function updateIndex(change: (entries: Map) => bool } catch (error) { if (!["EEXIST", "ENOTEMPTY", "EPERM", "EACCES"].includes(codeOf(error) ?? "")) throw error; await reapLock(lock); - if (performance.now() >= deadline) throw new Error(`Timed out waiting for plugin index lock: ${lock}`); + if (performance.now() >= deadline) + throw new Error(`Timed out waiting for plugin index lock: ${lock}`); await new Promise((done) => setTimeout(done, 25 + Math.random() * 50)); } } diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index 0852a744..a2b74cbd 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -19,34 +19,52 @@ function httpAddress(address: string) { return url; } +function optionalString(value: unknown): value is string | null | undefined { + return value === null || value === undefined || typeof value === "string"; +} + function discovery(value: unknown) { if (!value || typeof value !== "object") throw new Error("Invalid local server discovery"); const { pid, url, bundleHash, buildId, pluginRoot } = value as Record; if ( - typeof pid !== "number" || !processIsAlive(pid) || + typeof pid !== "number" || + !processIsAlive(pid) || typeof url !== "string" || - typeof bundleHash !== "string" || !/^[a-f0-9]{64}$/i.test(bundleHash) || - (buildId != null && typeof buildId !== "string") || - (pluginRoot != null && typeof pluginRoot !== "string") - ) throw new Error("Invalid local server identity"); + typeof bundleHash !== "string" || + !/^[a-f0-9]{64}$/i.test(bundleHash) || + !optionalString(buildId) || + !optionalString(pluginRoot) + ) + throw new Error("Invalid local server identity"); return { pid, url, bundleHash, buildId: buildId ?? null, pluginRoot: pluginRoot ?? null }; } -async function localEndpoint() { - const folder = dataFolder(); - const identity = discovery(JSON.parse(await readFile(join(folder, "server.json"), "utf8"))); - const ws = new URL(identity.url); - if ( - ws.protocol !== "ws:" || ws.username || ws.password || - !identity.url.endsWith("/ws") || ws.pathname !== "/ws" || ws.search || ws.hash || - !(ws.hostname === "[::1]" || (isIP(ws.hostname) === 4 && ws.hostname.startsWith("127."))) - ) throw new Error("Invalid local server discovery address"); +async function readToken(folder: string) { const file = join(folder, "server.token"); const metadata = await lstat(file); if (!metadata.isFile() || (process.platform !== "win32" && (metadata.mode & 0o077) !== 0)) throw new Error("Local server token must be a private regular file"); const token = await readFile(file, "utf8"); if (!/^[a-f0-9]{64}$/i.test(token)) throw new Error("Invalid local server token"); + return token; +} + +async function localEndpoint() { + const folder = dataFolder(); + const identity = discovery(JSON.parse(await readFile(join(folder, "server.json"), "utf8"))); + const ws = new URL(identity.url); + if ( + ws.protocol !== "ws:" || + ws.username || + ws.password || + !identity.url.endsWith("/ws") || + ws.pathname !== "/ws" || + ws.search || + ws.hash || + !(ws.hostname === "[::1]" || (isIP(ws.hostname) === 4 && ws.hostname.startsWith("127."))) + ) + throw new Error("Invalid local server discovery address"); + const token = await readToken(folder); const endpoint = { url: httpAddress(identity.url), token }; const health = discovery(await get("/health", endpoint)); if (JSON.stringify(health) !== JSON.stringify(identity)) throw new Error("Local server identity changed"); @@ -75,11 +93,15 @@ async function get(path: string, endpoint: { url: URL; token: string }): Promise export async function serverGet(path: string, explicit?: string): Promise { const address = explicit ?? process.env.TOKAMAK_SERVER; if (!address) { - try { - return await get(path, await localEndpoint()); - } catch { + const endpoint = await localEndpoint().catch(() => { throw new CommandError("No local server is running", "tokamak server start"); - } + }); + return get(path, endpoint).catch((error: Error) => { + throw new CommandError( + `Could not read ${path} from the local server: ${error.message}`, + "tokamak server start", + ); + }); } try { const token = process.env.TOKAMAK_TOKEN; diff --git a/packages/cli/tests/build.test.mjs b/packages/cli/tests/build.test.mjs new file mode 100644 index 00000000..a9acd4f4 --- /dev/null +++ b/packages/cli/tests/build.test.mjs @@ -0,0 +1,113 @@ +import { expect, test } from "bun:test"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { withPlugin, workspace } from "../../build/tests/outside-fixture.mjs"; +import { cli } from "./cli-fixture.mjs"; + +test("builds an external plugin and reports its output folder", async () => { + await withPlugin((plugin, root) => { + const result = cli(root, ["plugin", "build", plugin, "--json"]); + expect(result.code, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + ok: true, + id: "outside-fixture", + outputFolder: join(plugin, "dist"), + }); + }); +}, 120_000); + +test("returns structured type errors in both checker formats", async () => { + await withPlugin(async (plugin, root) => { + const file = join(plugin, "src/server/broken.ts"); + await writeFile(file, "type Box = { value: T };\nexport const broken: Box = 42;\n"); + for (const mode of ["plain", "tagged", "agent"]) { + const env = { + AI_AGENT: mode === "agent" ? "1" : undefined, + CLAUDECODE: mode === "tagged" ? "1" : undefined, + CLAUDE_CODE_ENTRYPOINT: mode === "tagged" ? "cli" : undefined, + }; + if (mode !== "agent") { + const raw = Bun.spawnSync([process.execPath, "packages/build/src/build.ts", plugin], { + cwd: workspace, + env: { ...process.env, ...env, TOKAMAK_SERVER_DIR: join(root, "data") }, + stdout: "pipe", + stderr: "pipe", + timeout: 60_000, + }); + const log = raw.stdout.toString() + raw.stderr.toString(); + expect(log).toMatch( + mode === "tagged" + ? /]*code="TS2322">/ + : /broken\.ts\(2,\d+\): error TS2322/, + ); + } + const result = cli(root, ["plugin", "build", plugin, "--json"], env); + expect(result.code).not.toBe(0); + const { error } = JSON.parse(result.stdout); + expect(error.message).toBe("Plugin build failed"); + expect(error.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + file, + line: 2, + column: expect.any(Number), + code: "TS2322", + message: "Type 'number' is not assignable to type 'Box'.", + }), + ]), + ); + } + }); +}, 120_000); + +test.each([ + [ + "missing import", + "src/server/broken.ts", + "import { x } from './not-there';\nexport { x };", + /not-there/, + 1, + ], + ["syntax error", "src/server/broken.ts", "export const broken = ;\n", /expected/i, 1], + ["window type error", "src/client/api.tsx", "export const broken: string = 42;\n", /not assignable/, 1], + [ + "stylesheet error", + "src/client/styles.css", + "@import 'tailwindcss';\n.bad { @apply no-such-utility; }\n", + /no-such-utility/, + null, + ], +])( + "returns a diagnostic for a %s", + async (_, relative, contents, message, line) => { + await withPlugin(async (plugin, root) => { + const file = join(plugin, relative); + await writeFile(file, contents); + const result = cli(root, ["plugin", "build", plugin, "--json"]); + expect(result.code).not.toBe(0); + const { error } = JSON.parse(result.stdout); + expect(error.message).toBe("Plugin build failed"); + const diagnostic = error.diagnostics.find((item) => message.test(item.message)); + expect(diagnostic).toBeDefined(); + if (line) expect(diagnostic).toMatchObject({ file, line, column: expect.any(Number) }); + else if (diagnostic.file) expect(diagnostic).toMatchObject({ file, line: expect.any(Number) }); + }); + }, + 120_000, +); + +test("build and add reject custom artifact output directories", async () => { + await withPlugin(async (plugin, root) => { + await writeFile(join(plugin, "vite.config.ts"), "export default { build: { outDir: 'custom-dist' } };\n"); + for (const command of ["build", "add"]) { + const result = cli(root, ["plugin", command, plugin, "--json"]); + expect(result.code).not.toBe(0); + const { error } = JSON.parse(result.stdout); + expect([error.message, ...error.diagnostics.map((entry) => entry.message)].join("\n")).toContain( + "must use", + ); + expect(error.diagnostics[0].message).toContain(join(plugin, "dist")); + } + expect(await Bun.file(join(root, "data", "plugin-index.json")).exists()).toBe(false); + }); +}, 120_000); diff --git a/packages/cli/tests/cli-fixture.mjs b/packages/cli/tests/cli-fixture.mjs new file mode 100644 index 00000000..87da6efb --- /dev/null +++ b/packages/cli/tests/cli-fixture.mjs @@ -0,0 +1,19 @@ +import { join } from "node:path"; +import { workspace } from "../../build/tests/outside-fixture.mjs"; + +export function cli(root, args, extra = {}) { + const result = Bun.spawnSync([process.execPath, "run", "--silent", "tokamak", ...args], { + cwd: workspace, + timeout: 60_000, + env: { + ...process.env, + TOKAMAK_SERVER: undefined, + TOKAMAK_TOKEN: undefined, + TOKAMAK_SERVER_DIR: join(root, "data"), + ...extra, + }, + stdout: "pipe", + stderr: "pipe", + }); + return { code: result.exitCode, stdout: result.stdout.toString(), stderr: result.stderr.toString() }; +} diff --git a/packages/cli/tests/cli.test.mjs b/packages/cli/tests/cli.test.mjs index ed880984..41703a4d 100644 --- a/packages/cli/tests/cli.test.mjs +++ b/packages/cli/tests/cli.test.mjs @@ -3,51 +3,7 @@ import { chmod, cp, mkdir, readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { randomBytes } from "node:crypto"; import { withPlugin, workspace } from "../../build/tests/outside-fixture.mjs"; - -function cli(root, args, extra = {}) { - const result = Bun.spawnSync([process.execPath, "run", "--silent", "tokamak", ...args], { - cwd: workspace, - timeout: 60_000, - env: { - ...process.env, - TOKAMAK_SERVER: undefined, - TOKAMAK_TOKEN: undefined, - TOKAMAK_SERVER_DIR: join(root, "data"), - ...extra, - }, - stdout: "pipe", - stderr: "pipe", - }); - return { code: result.exitCode, stdout: result.stdout.toString(), stderr: result.stderr.toString() }; -} - -test("builds an external plugin and reports its output folder", async () => { - await withPlugin((plugin, root) => { - const result = cli(root, ["plugin", "build", plugin, "--json"]); - expect(result.code, result.stderr).toBe(0); - expect(JSON.parse(result.stdout)).toEqual({ - ok: true, - id: "outside-fixture", - outputFolder: join(plugin, "dist"), - }); - }); -}, 120_000); - -test("returns structured type errors in both checker formats", async () => { - await withPlugin(async (plugin, root) => { - await writeFile(join(plugin, "src/server/broken.ts"), "\nexport const broken: string = 42;\n"); - for (const AI_AGENT of [undefined, "1"]) { - const result = cli(root, ["plugin", "build", plugin, "--json"], { AI_AGENT }); - expect(result.code).not.toBe(0); - const { error } = JSON.parse(result.stdout); - expect(error.diagnostics).toEqual( - expect.arrayContaining([ - expect.objectContaining({ file: expect.stringMatching(/broken\.ts$/), line: 2, code: "TS2322" }), - ]), - ); - } - }); -}, 120_000); +import { cli } from "./cli-fixture.mjs"; test("adds and removes folders and protects an existing id", async () => { await withPlugin(async (plugin, root) => { @@ -159,6 +115,7 @@ async function withServer(root, check) { cwd: root, env: { ...process.env, TOKAMAK_WEB_DIR: undefined }, stdout: "ignore", + timeout: 110_000, stderr: "pipe", }); let address; @@ -210,7 +167,7 @@ test("a real local server loads and removes an indexed plugin on restart", async run(["build", plugin]); await withServer(root, () => { expect(run(["list"])).toEqual({ ok: true, plugins: [] }); - run(["add", plugin]); + expect(run(["add", plugin]).precedence).toBe("index"); expect(run(["list"])).toEqual({ ok: true, plugins: [] }); }); await withServer(root, (address, token) => { @@ -242,6 +199,11 @@ test("rejects empty input and server selection on index commands without leaking expect(cli(root, ["plugin", "list", "--help"]).stdout).toContain("--server"); const list = cli(root, ["plugin", "list", "--server", address, "--json"], { TOKAMAK_TOKEN: "test" }); expect(list.code).not.toBe(0); + for (const malformed of [`--server=${address}`, address]) { + const rejected = cli(root, ["plugin", "list", malformed, "--json"]); + expect(rejected.code).not.toBe(0); + expect(rejected.stdout + rejected.stderr).not.toContain("private-password"); + } expect(list.stdout + list.stderr + add.stdout + add.stderr).not.toContain("private-password"); expect(Bun.file(join(root, "data", "plugin-index.json")).size).toBe(0); }); @@ -265,7 +227,7 @@ test("closed stdout ends quietly", async () => { }); expect(signal).toBeNull(); expect(code).toBe(0); - expect(stderr).not.toMatch(/EPIPE|stack trace|at .*\.ts:/); + expect(stderr).toBe(""); }); }, 60_000); @@ -278,24 +240,33 @@ test("concurrent adds preserve every update and recover an abandoned lock", asyn const expected = Object.fromEntries( Array.from({ length: 12 }, (_, i) => [`concurrent-${i}`, join(root, `concurrent-${i}`)]), ); - await Promise.all(Object.entries(expected).map(async ([id, folder]) => { - await mkdir(folder); - await writeFile(join(folder, "package.json"), JSON.stringify({ name: id })); - })); - await Promise.all(Object.values(expected).map(async (folder) => { - const child = Bun.spawn([process.execPath, "run", "--silent", "tokamak", "plugin", "add", folder, "--json"], { - cwd: workspace, - env: { ...process.env, TOKAMAK_SERVER_DIR: data }, - stdout: "pipe", - stderr: "pipe", - timeout: 60_000, - }); - const [code, stdout, stderr] = await Promise.all([ - child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(), - ]); - expect(code, stderr).toBe(0); - expect(JSON.parse(stdout).ok).toBe(true); - })); + await Promise.all( + Object.entries(expected).map(async ([id, folder]) => { + await mkdir(folder); + await writeFile(join(folder, "package.json"), JSON.stringify({ name: id })); + }), + ); + await Promise.all( + Object.values(expected).map(async (folder) => { + const child = Bun.spawn( + [process.execPath, "run", "--silent", "tokamak", "plugin", "add", folder, "--json"], + { + cwd: workspace, + env: { ...process.env, TOKAMAK_SERVER_DIR: data }, + stdout: "pipe", + stderr: "pipe", + timeout: 60_000, + }, + ); + const [code, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + expect(code, stderr).toBe(0); + expect(JSON.parse(stdout).ok).toBe(true); + }), + ); expect(JSON.parse(await readFile(join(data, "plugin-index.json"), "utf8"))).toEqual(expected); expect(await Bun.file(join(lock, "2147483647.abandoned")).exists()).toBe(false); }); @@ -307,19 +278,39 @@ test("stale discovery sends no token and mismatched health cannot supply a catal await mkdir(data); await writeFile(join(data, "server.token"), "a".repeat(64), { mode: 0o600 }); const requests = []; - const listener = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch(request) { - requests.push(new URL(request.url).pathname); - return Response.json({ plugins: [] }); - } }); + const listener = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + requests.push(new URL(request.url).pathname); + return Response.json({ plugins: [] }); + }, + }); const address = `http://127.0.0.1:${listener.port}`; - const identity = { pid: 2147483647, url: `ws://127.0.0.1:${listener.port}/ws`, bundleHash: "b".repeat(64) }; + const identity = { + pid: 2147483647, + url: `ws://127.0.0.1:${listener.port}/ws`, + bundleHash: "b".repeat(64), + }; async function list(env = {}) { const child = Bun.spawn([process.execPath, "run", "--silent", "tokamak", "plugin", "list", "--json"], { cwd: workspace, - env: { ...process.env, TOKAMAK_SERVER: undefined, TOKAMAK_TOKEN: undefined, TOKAMAK_SERVER_DIR: data, ...env }, - stdout: "pipe", stderr: "pipe", timeout: 30_000, + env: { + ...process.env, + TOKAMAK_SERVER: undefined, + TOKAMAK_TOKEN: undefined, + TOKAMAK_SERVER_DIR: data, + ...env, + }, + stdout: "pipe", + stderr: "pipe", + timeout: 30_000, }); - const [code, stdout] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]); + const [code, stdout] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); expect(code).not.toBe(0); return JSON.parse(stdout).error; } diff --git a/packages/sdk/tools/bundle.mjs b/packages/sdk/tools/bundle.mjs index 121c73c6..051bbf84 100644 --- a/packages/sdk/tools/bundle.mjs +++ b/packages/sdk/tools/bundle.mjs @@ -161,16 +161,12 @@ if (import.meta.main) { async function scanFiles(root) { const files = []; - const scanner = new Bun.Transpiler({ loader: "ts" }); for await (const scanned of new Bun.Glob("**/*.ts").scan({ cwd: root, onlyFiles: true, })) { const file = scanned.split(sep).join("/"); if (!isServerModule(root, file)) continue; - // scan is a real parser, not a regexp. We do not interpret initializer - // syntax: exported definitions are recognized by their runtime brand. - scanner.scan(await Bun.file(join(root, file)).text()); files.push(file); } files.sort(); diff --git a/packages/server/src/plugins/catalog.rs b/packages/server/src/plugins/catalog.rs index 11556349..9a8f3236 100644 --- a/packages/server/src/plugins/catalog.rs +++ b/packages/server/src/plugins/catalog.rs @@ -6,80 +6,27 @@ impl Catalog { } pub(crate) fn read_reusing(root: &Path, previous: Option<&Self>) -> Result { - Self::read_in_directory( - root, - previous, - previous - .map(|p| p.directory.as_path()) - .unwrap_or(root.parent().unwrap_or(root)), - None, - ) + let directory = previous + .map(|p| p.directory.as_path()) + .unwrap_or(root.parent().unwrap_or(root)); + let mut plugins = read_installed(root, previous)?; + let index = previous.map(|p| Arc::clone(&p.index)).unwrap_or_default(); + for (id, folder) in index.iter() { + let old = previous.and_then(|p| p.plugins.get(id)); + plugins.insert( + id.clone(), + read_indexed(id, folder, old.map(AsRef::as_ref))?, + ); + } + Self::assemble(root, directory, plugins, index) } - pub(crate) fn read_in_directory( + pub(super) fn assemble( root: &Path, - previous: Option<&Self>, directory: &Path, - index_path: Option<&Path>, + plugins: BTreeMap>, + index: Arc>, ) -> Result { - let directories = if let Some(directory) = artifact_directory(root) { - vec![directory] - } else { - let mut result = Vec::new(); - - for entry in std::fs::read_dir(root).map_err(|e| invalid(e.to_string()))? { - let entry = entry.map_err(|e| invalid(e.to_string()))?; - - if entry.file_name().to_string_lossy().starts_with('.') { - continue; - } - - let path = entry.path(); - - if let Some(directory) = artifact_directory(&path) { - result.push(directory); - } else if path.join("package.json").is_file() { - return Err(invalid(format!( - "Plugin has not been built: {}", - path.display() - ))); - } - } - result.sort(); - result - }; - - // Keep index membership fixed for this server lifetime, even across a - // reload through the server's existing plugin controls. - let index = match previous { - Some(catalog) => Arc::clone(&catalog.index), - None => Arc::new(read_index(index_path)?), - }; - let mut sources = BTreeMap::new(); - for directory in directories { - let id = Manifest::read(&directory)?.plugin.id; - if sources.insert(id.clone(), directory).is_some() { - return Err(invalid(format!( - "Duplicate installed plugin identity: {id}" - ))); - } - } - - let indexed = select_indexed_sources(&index, &mut sources)?; - - let mut plugins = BTreeMap::new(); - for (id, directory) in sources { - let old = previous.and_then(|p| p.plugins.get(&id)); - let artifact = Artifact::read_reusing(&directory, old.map(AsRef::as_ref))?; - if artifact.manifest.plugin.id != id { - return Err(invalid(format!( - "Plugin identity changed while reading {}", - directory.display() - ))); - } - plugins.insert(id, Arc::new(artifact)); - } - let order = resolve(&plugins)?; let mut hash = Sha256::new(); @@ -109,7 +56,6 @@ impl Catalog { revision: format!("{:x}", hash.finalize()), providers, index, - indexed, environment, directory: directory.to_owned(), }) @@ -119,7 +65,7 @@ impl Catalog { json!({"revision":self.revision,"plugins":self.order.iter().map(|id| { let manifest = &self.plugins[id].manifest; let mut value = serde_json::to_value(manifest).expect("manifest serialization"); - value["fromIndex"] = json!(self.indexed.contains(id)); + value["fromIndex"] = json!(self.index.contains_key(id)); value["revision"] = json!(self.revision); value["uiUrl"] = manifest.entries.ui.as_ref().map(|ui| format!("/plugins/{id}/{}/assets/{}", self.revision, ui.code)).into(); value["stylesheetUrl"] = manifest.entries.ui.as_ref().and_then(|ui| ui.css.as_ref()).map(|css| format!("/plugins/{id}/{}/assets/{css}", self.revision)).into(); @@ -178,56 +124,109 @@ fn artifact_directory(root: &Path) -> Option { } } -fn read_index(path: Option<&Path>) -> Result, Error> { +pub(super) type IndexedArtifacts = BTreeMap)>; + +pub(super) fn read_index(path: Option<&Path>) -> IndexedArtifacts { let Some(path) = path else { - return Ok(BTreeMap::new()); + return BTreeMap::new(); + }; + let entries = match index_entries(path) { + Ok(entries) => entries, + Err(error) => { + eprintln!("Warning: ignoring plugin index {}: {error}", path.display()); + return BTreeMap::new(); + } }; + entries + .into_iter() + .filter_map(|(id, value)| { + let folder = Path::new(value.as_str().unwrap_or_default()); + let result = if value.is_string() { + read_indexed(&id, folder, None) + } else { + Err(invalid("Expected an absolute source folder string")) + }; + match result { + Ok(artifact) => Some((id, (folder.to_owned(), artifact))), + Err(error) => { + let source = value + .as_str() + .map(str::to_owned) + .unwrap_or_else(|| value.to_string()); + eprintln!("Skipping indexed plugin {id} at {source}: {error}"); + None + } + } + }) + .collect() +} + +fn index_entries(path: &Path) -> Result, Error> { let source = match std::fs::read(path) { Ok(source) => source, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(BTreeMap::new()), - Err(error) => { - return Err(invalid(format!( - "Cannot read plugin index {}: {error}", - path.display() - ))); - } + Err(error) => return Err(invalid(error.to_string())), }; - let entries: BTreeMap = serde_json::from_slice(&source) - .map_err(|error| invalid(format!("Invalid plugin index {}: {error}", path.display())))?; - for (id, folder) in &entries { - if !folder.is_absolute() { - return Err(invalid(format!( - "Indexed plugin {id} needs an absolute source folder" - ))); - } + serde_json::from_slice(&source).map_err(|error| invalid(error.to_string())) +} + +fn read_indexed(id: &str, folder: &Path, old: Option<&Artifact>) -> Result, Error> { + if !folder.is_absolute() { + return Err(invalid("Expected an absolute source folder")); + } + let dist = folder.join("dist"); + let artifact = Artifact::read_reusing(&dist, old)?; + if artifact.manifest.plugin.id != id { + return Err(invalid(format!( + "Build has plugin identity {}", + artifact.manifest.plugin.id + ))); } - Ok(entries) + Ok(Arc::new(artifact)) } -fn select_indexed_sources( - index: &BTreeMap, - sources: &mut BTreeMap, -) -> Result, Error> { - let mut indexed = BTreeSet::new(); - for (id, folder) in index { - let dist = folder.join("dist"); - if !dist.join("manifest.json").is_file() { - eprintln!( - "Skipping indexed plugin {id}: no build at {}", - dist.display() - ); - continue; +pub(super) fn read_installed( + root: &Path, + previous: Option<&Catalog>, +) -> Result>, Error> { + let directories = if let Some(directory) = artifact_directory(root) { + vec![directory] + } else { + let mut result = Vec::new(); + for entry in std::fs::read_dir(root).map_err(|error| invalid(error.to_string()))? { + let entry = entry.map_err(|error| invalid(error.to_string()))?; + if entry.file_name().to_string_lossy().starts_with('.') { + continue; + } + let path = entry.path(); + if let Some(directory) = artifact_directory(&path) { + result.push(directory); + } else if path.join("package.json").is_file() { + return Err(invalid(format!( + "Plugin has not been built: {}", + path.display() + ))); + } + } + result.sort(); + result + }; + let mut plugins = BTreeMap::new(); + for directory in directories { + let id = Manifest::read(&directory)?.plugin.id; + let old = previous.and_then(|p| p.plugins.get(&id)); + let artifact = Artifact::read_reusing(&directory, old.map(AsRef::as_ref))?; + if artifact.manifest.plugin.id != id { + return Err(invalid(format!( + "Plugin identity changed while reading {}", + directory.display() + ))); } - let actual = Manifest::read(&dist)?.plugin.id; - if actual != *id { + if plugins.insert(id.clone(), Arc::new(artifact)).is_some() { return Err(invalid(format!( - "Indexed plugin {id} has plugin identity {actual} in {}", - dist.display() + "Duplicate installed plugin identity: {id}" ))); } - sources.insert(id.clone(), dist); - indexed.insert(id.clone()); } - - Ok(indexed) + Ok(plugins) } diff --git a/packages/server/src/plugins/grants.rs b/packages/server/src/plugins/grants.rs index baaea3af..dc7980c5 100644 --- a/packages/server/src/plugins/grants.rs +++ b/packages/server/src/plugins/grants.rs @@ -12,6 +12,22 @@ impl Loaded { &self, database: &crate::data::Database, previous: Option<&Self>, + ) -> Result<(), Error> { + self.apply_grants(database, previous, true).await + } + + pub(super) async fn check_startup_grants( + &self, + database: &crate::data::Database, + ) -> Result<(), Error> { + self.apply_grants(database, None, false).await + } + + async fn apply_grants( + &self, + database: &crate::data::Database, + previous: Option<&Self>, + commit: bool, ) -> Result<(), Error> { let table = DataSpace::plugin("tokamak.host") .map_err(|e| invalid(e.to_string()))? @@ -52,6 +68,9 @@ impl Loaded { .await; match result { + Ok(_) if !commit => Box::pin(tx.rollback()) + .await + .map_err(|error| invalid(error.to_string())), Ok((granted, resets)) => { Box::pin(tx.commit()) .await diff --git a/packages/server/src/plugins/loading.rs b/packages/server/src/plugins/loading.rs index a8a56a41..6f3e3a82 100644 --- a/packages/server/src/plugins/loading.rs +++ b/packages/server/src/plugins/loading.rs @@ -1,6 +1,41 @@ -use super::*; +use super::{ + catalog::{IndexedArtifacts, read_index, read_installed}, + *, +}; impl Loaded { + pub async fn startup( + root: &Path, + directory: &Path, + index: Option<&Path>, + pool: &engine::Pool, + database: &crate::data::Database, + ) -> Result, Error> { + let installed = read_installed(root, None)?; + let candidates = read_index(index); + let config = StartupCatalog { + root, + directory, + pool, + database, + }; + let loaded = match config.commit(&installed, &candidates, None).await { + Ok(loaded) => loaded, + Err(error) if candidates.is_empty() => return Err(error), + Err(_) => config.recover(&installed, candidates).await?, + }; + for (id, folder) in loaded.catalog.index.iter() { + if installed.contains_key(id) { + eprintln!( + "Indexed plugin {id} at {} replaces the copy from --plugins", + folder.display() + ); + } + } + Ok(loaded) + } + + #[cfg(test)] pub async fn load(catalog: Catalog, pool: &engine::Pool) -> Result, Error> { Self::load_reusing(catalog, pool, None).await } @@ -35,7 +70,9 @@ impl Loaded { let (descriptor, kinds) = match cached { Some(value) => value.clone(), - None => crate::service::inspect(pool, bundle.clone()).await?, + None => crate::service::inspect(pool, bundle.clone()) + .await + .map_err(|error| invalid(format!("Could not inspect plugin {id}: {error}")))?, }; inspected.insert(id.clone(), (descriptor.clone(), kinds.clone())); @@ -202,7 +239,10 @@ fn start_workers( catalog.directory.join("plugin-data").join(id), &catalog.environment, &artifact.manifest.plugin.permissions, - )?, + ) + .map_err(|error| { + invalid(format!("Could not set up worker for plugin {id}: {error}")) + })?, ); } } @@ -226,3 +266,93 @@ fn validate_maintenance( } Ok(()) } + +struct StartupCatalog<'a> { + root: &'a Path, + directory: &'a Path, + pool: &'a engine::Pool, + database: &'a crate::data::Database, +} + +impl StartupCatalog<'_> { + async fn load( + &self, + installed: &BTreeMap>, + indexed: &IndexedArtifacts, + previous: Option<&Loaded>, + ) -> Result, Error> { + let mut plugins = installed.clone(); + let index = indexed + .iter() + .map(|(id, (folder, artifact))| { + plugins.insert(id.clone(), Arc::clone(artifact)); + (id.clone(), folder.clone()) + }) + .collect(); + let catalog = Catalog::assemble(self.root, self.directory, plugins, Arc::new(index))?; + Loaded::load_reusing(catalog, self.pool, previous).await + } + + async fn commit( + &self, + installed: &BTreeMap>, + indexed: &IndexedArtifacts, + previous: Option<&Loaded>, + ) -> Result, Error> { + let loaded = self.load(installed, indexed, previous).await?; + Box::pin(loaded.record_grants(self.database, None)).await?; + Ok(loaded) + } + + async fn trial( + &self, + installed: &BTreeMap>, + indexed: &IndexedArtifacts, + previous: &Loaded, + ) -> Result, Error> { + let loaded = self.load(installed, indexed, Some(previous)).await?; + // Run the same schema and grant checks as startup without publishing a + // partially accepted generation or making persistent changes. + Box::pin(loaded.check_startup_grants(self.database)).await?; + Ok(loaded) + } + + async fn recover( + &self, + installed: &BTreeMap>, + mut pending: IndexedArtifacts, + ) -> Result, Error> { + let mut accepted = BTreeMap::new(); + let mut loaded = self.load(installed, &accepted, None).await?; + let mut failures = BTreeMap::new(); + // Retry failed candidates after a successful pass: a dependency may + // have been accepted later in the order. Keep each installed fallback + // until its replacement has passed the complete loading pipeline. + loop { + let remaining = pending.len(); + for (id, entry) in std::mem::take(&mut pending) { + accepted.insert(id.clone(), entry.clone()); + match self.trial(installed, &accepted, &loaded).await { + Ok(next) => loaded = next, + Err(error) => { + accepted.remove(&id); + pending.insert(id.clone(), entry); + failures.insert(id, error); + } + } + } + if pending.is_empty() || pending.len() == remaining { + break; + } + } + for (id, (folder, _)) in pending { + if let Some(error) = failures.get(&id) { + eprintln!( + "Skipping indexed plugin {id} at {}: {error}", + folder.display() + ); + } + } + self.commit(installed, &accepted, Some(&loaded)).await + } +} diff --git a/packages/server/src/plugins/mod.rs b/packages/server/src/plugins/mod.rs index 6620de17..3fe02d36 100644 --- a/packages/server/src/plugins/mod.rs +++ b/packages/server/src/plugins/mod.rs @@ -27,7 +27,6 @@ pub struct Catalog { pub revision: String, providers: BTreeMap>, index: Arc>, - indexed: BTreeSet, pub(crate) environment: PathBuf, pub(crate) directory: PathBuf, } diff --git a/packages/server/src/service/startup.rs b/packages/server/src/service/startup.rs index d281cf19..1ee604e0 100644 --- a/packages/server/src/service/startup.rs +++ b/packages/server/src/service/startup.rs @@ -95,18 +95,8 @@ impl Server { let directory = database .directory() .unwrap_or(root.parent().unwrap_or(root)); - let loaded = match crate::plugins::Catalog::read_in_directory(root, None, directory, index) - { - Ok(catalog) => crate::plugins::Loaded::load(catalog, &pool).await, - Err(error) => Err(error), - }; - - let loaded = match loaded { - Ok(loaded) => Box::pin(loaded.record_grants(&database, None)) - .await - .map(|()| loaded), - Err(error) => Err(error), - }; + let loaded = + crate::plugins::Loaded::startup(root, directory, index, &pool, &database).await; let loaded = match loaded { Ok(loaded) => loaded, diff --git a/packages/server/tests/plugins.rs b/packages/server/tests/plugins.rs index 06d1444e..ee346f27 100644 --- a/packages/server/tests/plugins.rs +++ b/packages/server/tests/plugins.rs @@ -254,7 +254,7 @@ fn standalone_binary_exposes_only_authenticated_plugin_controls_and_cli() { "plugin server exited before readiness" ); assert!( - started.elapsed() < Duration::from_secs(20), + started.elapsed() < Duration::from_secs(90), "plugin startup timed out" ); std::thread::sleep(Duration::from_millis(20)); @@ -958,6 +958,7 @@ async fn start_index_test_server(local: &std::path::Path, installed: &std::path: .arg("--plugins") .arg(installed) .env_remove("TOKAMAK_WEB_DIR") + .env_remove("TKM167_WORKER_TEST_MISSING") .spawn() .unwrap(), ); @@ -968,10 +969,194 @@ async fn start_index_test_server(local: &std::path::Path, installed: &std::path: "server exited before readiness" ); assert!( - started.elapsed() < Duration::from_secs(20), + started.elapsed() < Duration::from_secs(90), "startup timed out" ); tokio::time::sleep(Duration::from_millis(20)).await; } process } + +#[tokio::test] +async fn local_server_falls_back_from_a_malformed_index() { + let installed = Installation::new(); + let temporary = tempfile::tempdir().unwrap(); + let index = temporary.path().join("plugin-index.json"); + std::fs::write(&index, "{broken").unwrap(); + let catalog = index_test_catalog(temporary.path(), &installed.0).await; + assert_eq!(catalog["plugins"].as_array().unwrap().len(), 2); + assert!( + catalog["plugins"] + .as_array() + .unwrap() + .iter() + .all(|p| p["fromIndex"] == false) + ); +} + +#[tokio::test] +async fn local_server_keeps_installed_plugins_and_other_index_entries_when_candidates_fail() { + let installed = Installation::new(); + let temporary = tempfile::tempdir().unwrap(); + let local = temporary.path(); + let bad_override = index_test_artifact(local, &installed, "bad-override", "a", |manifest| { + manifest["plugin"]["depends"] = json!({"unavailable":"*"}); + }); + let wrong_id = index_test_artifact(local, &installed, "wrong-id", "wrong-id", |_| {}); + let malformed = local.join("malformed"); + std::fs::create_dir_all(malformed.join("dist")).unwrap(); + std::fs::write(malformed.join("dist/manifest.json"), "bad JSON").unwrap(); + let module = index_test_artifact(local, &installed, "module", "invalid-module", |manifest| { + manifest["hashes"]["bundle.js"] = json!(format!( + "{:x}", + Sha256::digest(b"throw new Error('broken module');") + )); + }); + std::fs::write( + module.join("dist/bundle.js"), + "throw new Error('broken module');", + ) + .unwrap(); + let valid = index_test_artifact(local, &installed, "valid", "valid", |_| {}); + std::fs::write( + local.join("plugin-index.json"), + json!({ "a": bad_override, "relative": "./bad", "mismatch": wrong_id, "malformed": malformed, + "invalid-module": module, "valid": valid }).to_string(), + ).unwrap(); + let catalog = index_test_catalog(local, &installed.0).await; + let plugins = catalog["plugins"].as_array().unwrap(); + assert_eq!(plugins.len(), 3); + for (id, from_index) in [("a", false), ("b", false), ("valid", true)] { + assert_eq!( + plugins + .iter() + .find(|plugin| plugin["plugin"]["id"] == id) + .unwrap()["fromIndex"], + from_index + ); + } +} + +fn index_test_artifact( + local: &std::path::Path, + installed: &Installation, + folder: &str, + id: &str, + edit: impl FnOnce(&mut Value), +) -> PathBuf { + let folder = local.join(folder); + let dist = folder.join("dist"); + std::fs::create_dir_all(&dist).unwrap(); + for file in ["manifest.json", "bundle.js", "bundle.js.map", "api.ts"] { + std::fs::copy(installed.0.join("a").join(file), dist.join(file)).unwrap(); + } + let manifest = dist.join("manifest.json"); + let mut value: Value = serde_json::from_slice(&std::fs::read(&manifest).unwrap()).unwrap(); + value["plugin"]["id"] = json!(id); + edit(&mut value); + std::fs::write(manifest, value.to_string()).unwrap(); + folder +} + +async fn index_test_catalog(local: &std::path::Path, installed: &std::path::Path) -> Value { + let mut process = start_index_test_server(local, installed).await; + let identity: Value = + serde_json::from_slice(&std::fs::read(local.join("server.json")).unwrap()).unwrap(); + let url = identity["url"] + .as_str() + .unwrap() + .replace("ws://", "http://"); + let base = url.trim_end_matches("/ws"); + let http = reqwest::Client::builder() + .timeout(Duration::from_secs(20)) + .build() + .unwrap(); + let token = "b".repeat(64); + let body = http + .get(format!("{base}/plugins")) + .bearer_auth(&token) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .bytes() + .await + .unwrap(); + if let Ok(reply) = http + .post(format!("{base}/shutdown")) + .bearer_auth(token) + .send() + .await + { + reply.error_for_status().unwrap(); + } + let status = tokio::time::timeout(Duration::from_secs(30), async { + loop { + if let Some(status) = process.0.try_wait().unwrap() { + break status; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap(); + assert!(status.success()); + serde_json::from_slice(&body).unwrap() +} + +#[tokio::test] +async fn local_server_skips_an_indexed_plugin_when_worker_setup_fails() { + let fixture = Installation::new(); + let temporary = tempfile::tempdir().unwrap(); + let local = temporary.path(); + let installed = local.join("installed"); + std::fs::create_dir(&installed).unwrap(); + let worker = index_test_artifact(local, &fixture, "worker", "worker", |manifest| { + manifest["plugin"]["depends"] = json!({}); + manifest["plugin"]["runtime"] = json!("worker"); + manifest["plugin"]["permissions"] = json!(["env:TKM167_WORKER_TEST_MISSING"]); + manifest["entries"]["worker"] = + json!({ "entry": "worker-entry.js", "code": "worker.js", "map": "worker.js.map" }); + for file in ["worker-entry.js", "worker.js", "worker.js.map"] { + manifest["hashes"][file] = json!(format!("{:x}", Sha256::digest(b"test"))); + } + }); + for file in ["worker-entry.js", "worker.js", "worker.js.map"] { + std::fs::write(worker.join("dist").join(file), "test").unwrap(); + } + let valid = index_test_artifact(local, &fixture, "valid", "valid", |manifest| { + manifest["plugin"]["depends"] = json!({}); + }); + std::fs::write(local.join(".env.local"), "invalid environment file !").unwrap(); + std::fs::write( + local.join("plugin-index.json"), + json!({ "worker": worker, "valid": valid }).to_string(), + ) + .unwrap(); + let catalog = index_test_catalog(local, &installed).await; + let plugins = catalog["plugins"].as_array().unwrap(); + assert_eq!(plugins.len(), 1); + assert_eq!(plugins[0]["plugin"]["id"], "valid"); +} + +#[tokio::test] +async fn local_server_keeps_an_installed_copy_when_indexed_grants_cannot_be_recorded() { + let installed = Installation::new(); + let temporary = tempfile::tempdir().unwrap(); + let local = temporary.path(); + let first = index_test_catalog(local, &installed.0).await; + assert_eq!(first["plugins"].as_array().unwrap().len(), 2); + let replacement = index_test_artifact(local, &installed, "replacement", "a", |manifest| { + manifest["plugin"]["permissions"] = json!(["env:TKM167_TEST_GRANT"]); + }); + std::fs::write( + local.join("plugin-index.json"), + json!({ "a": replacement }).to_string(), + ) + .unwrap(); + let catalog = index_test_catalog(local, &installed.0).await; + let plugins = catalog["plugins"].as_array().unwrap(); + assert_eq!(plugins.len(), 2); + assert!(plugins.iter().all(|plugin| plugin["fromIndex"] == false)); +} From 07b139d3ec4f69bb2b13cd18ba8ba8f2d9f37e6d Mon Sep 17 00:00:00 2001 From: JustKira Date: Fri, 20 Nov 2026 17:18:33 +0200 Subject: [PATCH 7/8] Use direct sockets for loopback CLI requests --- packages/cli/src/server.ts | 17 +++--- packages/cli/tests/discovery.test.mjs | 80 ++++++++++++++++++++++++++ packages/runtime/tools/direct-http.ts | 32 +++++++++++ packages/runtime/tools/local-server.ts | 36 +++--------- 4 files changed, 126 insertions(+), 39 deletions(-) create mode 100644 packages/cli/tests/discovery.test.mjs create mode 100644 packages/runtime/tools/direct-http.ts diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index 28fe1a62..e7ccf739 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -1,14 +1,11 @@ import { lstat, readFile } from "node:fs/promises"; import { isIP } from "node:net"; import { join } from "node:path"; +import { direct, loopback } from "../../runtime/tools/direct-http"; import { CommandError } from "./command"; import { dataFolder } from "./data"; import { processIsAlive } from "./process"; -const loopback = (host: string) => - host === "localhost" || host === "[::1]" || (isIP(host) === 4 && host.startsWith("127.")); -const noProxy = { proxy: "" }; - function httpAddress(address: string) { let url: URL; try { @@ -78,12 +75,12 @@ async function localEndpoint() { async function get(path: string, endpoint: { url: URL; token: string }): Promise { let response: Response; try { - response = await fetch(new URL(path, endpoint.url), { - ...(loopback(endpoint.url.hostname) ? noProxy : {}), - headers: { Authorization: `Bearer ${endpoint.token}` }, - signal: AbortSignal.timeout(5000), - redirect: "error", - }); + const url = new URL(path, endpoint.url); + const headers = new Headers({ Authorization: `Bearer ${endpoint.token}` }); + const signal = AbortSignal.timeout(5000); + response = loopback(url.hostname) + ? await direct(url, headers, signal, {}) + : await fetch(url, { headers, signal, redirect: "error" }); } catch { throw new Error("Could not connect to the server"); } diff --git a/packages/cli/tests/discovery.test.mjs b/packages/cli/tests/discovery.test.mjs new file mode 100644 index 00000000..f2cae1ab --- /dev/null +++ b/packages/cli/tests/discovery.test.mjs @@ -0,0 +1,80 @@ +import { expect, test } from "bun:test"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { withPlugin, workspace } from "../../build/tests/outside-fixture.mjs"; + +test("discovered and explicit loopback requests bypass environment proxies", async () => { + await withPlugin(async (_, root) => { + const received = []; + let proxyRequests = 0; + const token = "a".repeat(64); + const catalog = { plugins: [] }; + const listener = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + if (request.headers.get("authorization") !== `Bearer ${token}`) { + return new Response(null, { status: 401 }); + } + const path = new URL(request.url).pathname; + received.push(path); + return Response.json(path === "/health" ? identity : catalog); + }, + }); + const proxy = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch() { + proxyRequests++; + return new Response(null, { status: 502 }); + }, + }); + const data = join(root, "data"); + const address = `http://127.0.0.1:${listener.port}`; + const proxyUrl = `http://127.0.0.1:${proxy.port}`; + const identity = { + pid: process.pid, + url: `ws://127.0.0.1:${listener.port}/ws`, + bundleHash: "b".repeat(64), + }; + try { + await mkdir(data); + await writeFile(join(data, "server.json"), JSON.stringify(identity)); + await writeFile(join(data, "server.token"), token, { mode: 0o600 }); + for (const explicit of [false, true]) { + const child = Bun.spawn([process.execPath, "packages/cli/src/main.ts", "plugin", "list", "--json"], { + cwd: workspace, + env: { + ...process.env, + TOKAMAK_SERVER_DIR: data, + TOKAMAK_SERVER: explicit ? address : undefined, + TOKAMAK_TOKEN: explicit ? token : undefined, + HTTP_PROXY: proxyUrl, + HTTPS_PROXY: proxyUrl, + ALL_PROXY: proxyUrl, + http_proxy: proxyUrl, + https_proxy: proxyUrl, + all_proxy: proxyUrl, + NO_PROXY: "", + no_proxy: "", + }, + stdout: "pipe", + stderr: "pipe", + timeout: 30_000, + }); + const [code, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + expect(code, stderr).toBe(0); + expect(JSON.parse(stdout)).toEqual({ ok: true, ...catalog }); + } + expect(received).toEqual(["/health", "/plugins", "/plugins"]); + expect(proxyRequests).toBe(0); + } finally { + listener.stop(true); + proxy.stop(true); + } + }); +}); diff --git a/packages/runtime/tools/direct-http.ts b/packages/runtime/tools/direct-http.ts new file mode 100644 index 00000000..dfa2eee0 --- /dev/null +++ b/packages/runtime/tools/direct-http.ts @@ -0,0 +1,32 @@ +import { request as httpRequest } from "node:http"; +import { request as httpsRequest } from "node:https"; +import { isIP } from "node:net"; +import { Readable } from "node:stream"; + +export const loopback = (host: string) => + host === "localhost" || host === "[::1]" || (isIP(host) === 4 && host.startsWith("127.")); + +export type RequestOptions = { method?: string; headers?: Record; body?: string }; + +// A fresh socket cannot inherit proxy settings or follow redirects. +export function direct(url: URL, headers: Headers, signal: AbortSignal, { method, body }: RequestOptions) { + return new Promise((resolve, reject) => { + const send = url.protocol === "https:" ? httpsRequest : httpRequest; + const outgoing = send( + url, + { agent: false, method, headers: Object.fromEntries(headers), signal }, + (incoming) => { + const status = incoming.statusCode ?? 500; + const bodyless = status === 204 || status === 205 || status === 304; + if (bodyless) incoming.resume(); + resolve( + new Response(bodyless ? null : (Readable.toWeb(incoming) as ReadableStream), { + status, + }), + ); + }, + ); + outgoing.on("error", reject); + outgoing.end(body); + }); +} diff --git a/packages/runtime/tools/local-server.ts b/packages/runtime/tools/local-server.ts index 578e439b..9fa80825 100644 --- a/packages/runtime/tools/local-server.ts +++ b/packages/runtime/tools/local-server.ts @@ -2,9 +2,8 @@ import { createHash, randomBytes } from "node:crypto"; import { spawn, spawnSync } from "node:child_process"; import { link, lstat, mkdir, open, readFile, rm, realpath, writeFile } from "node:fs/promises"; -import { request } from "node:http"; -import { Readable } from "node:stream"; import { resolve } from "node:path"; +import { direct } from "./direct-http"; export type Identity = { pid: number; @@ -54,30 +53,9 @@ function endpoint(identity: Identity, path: string): URL { return url; } -// A one-use HTTP connection cannot inherit an environment proxy or follow a redirect. -function direct(url: URL, capability: string, timeout: number, method = "GET"): Promise { - return new Promise((resolve, reject) => { - const outgoing = request( - url, - { - agent: false, - method, - signal: AbortSignal.timeout(timeout), - headers: { authorization: `Bearer ${capability}` }, - }, - (incoming) => { - const code = incoming.statusCode ?? 500; - const bodyless = code === 204 || code === 205 || code === 304; - if (bodyless) incoming.resume(); - resolve( - new Response(bodyless ? null : (Readable.toWeb(incoming) as ReadableStream), { - status: code, - }), - ); - }, - ); - outgoing.on("error", reject); - outgoing.end(); +function localRequest(url: URL, capability: string, timeout: number, method = "GET") { + return direct(url, new Headers({ authorization: `Bearer ${capability}` }), AbortSignal.timeout(timeout), { + method, }); } @@ -121,7 +99,7 @@ export async function status(directory: string): Promise { const capability = await token(directory); let response: Response; try { - response = await direct(url, capability, 1000); + response = await localRequest(url, capability, 1000); } catch { return; } @@ -145,7 +123,7 @@ export async function status(directory: string): Promise { export async function fetchLocal(directory: string, path: string): Promise { const identity = await status(directory); if (!identity) return; - return direct(endpoint(identity, path), await token(directory), 3000); + return localRequest(endpoint(identity, path), await token(directory), 3000); } async function wait(): Promise { @@ -211,7 +189,7 @@ export async function stop(directory: string): Promise { const capability = await token(directory); let response: Response | undefined; try { - response = await direct(url, capability, 5000, "POST"); + response = await localRequest(url, capability, 5000, "POST"); } catch { // Cancellation closes accepted HTTP I/O as well as WebSockets. A lost // acknowledgement is not proof of shutdown: still await the readiness From 6165ba5a4f83bdb9ee7e1fa13942af38205405aa Mon Sep 17 00:00:00 2001 From: JustKira Date: Fri, 20 Nov 2026 17:23:20 +0200 Subject: [PATCH 8/8] Allow TypeScript source imports in e2e checks --- e2e/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/tsconfig.json b/e2e/tsconfig.json index 76ed5fc2..33af96a0 100644 --- a/e2e/tsconfig.json +++ b/e2e/tsconfig.json @@ -7,7 +7,8 @@ }, "lib": ["ES2022", "DOM"], "skipLibCheck": true, - "allowJs": true + "allowJs": true, + "allowImportingTsExtensions": true }, "include": [ "run.ts",