Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions GLOSSARY.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ _Avoid_: Hot swap, refresh, redeploy
The window drops a plugin's views and mounts them again from the new code. The window itself stays open, and nothing inside the views is kept.
_Avoid_: Refresh, window reload

**Package**:
One ZIP containing a built plugin’s manifest and the files it lists.
_Avoid_: Source archive

**Build id**:
The SHA-256 of a built plugin's `manifest.json`. The manifest holds the hash of every other file, so the build id names one exact build. The version is only a label.
_Avoid_: Revision, hash, version
Expand Down
5 changes: 5 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions packages/build/src/artifact-bundle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { definePlugin, type PluginDefinition } from "@tokamak/sdk";
import { uiContributions } from "./contributions.ts";
import { createHash } from "node:crypto";
import type { Plugin } from "vite";
import { tokamakVersion } from "../../tokamak-version";

type GenerateBundle = Extract<NonNullable<Plugin["generateBundle"]>, (...args: never[]) => unknown>;

Expand Down Expand Up @@ -48,6 +49,7 @@ export function artifactBundle(
...(bundle["styles.css"] ? { css: "styles.css" } : {}),
};
}
metadata.plugin = { ...metadata.plugin, api: `tokamak@${tokamakVersion()}` };
const hashes = bundleHashes(bundle);
manifest.source = JSON.stringify({ ...metadata, hashes }, null, 2) + "\n";
};
Expand Down
63 changes: 63 additions & 0 deletions packages/build/src/artifact-sources.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path";
import { fileURLToPath } from "node:url";
import ts from "typescript";

const packages = fileURLToPath(new URL("../../", import.meta.url));
const portable = (path: string) => path.split(sep).join("/");
const syntheticEntry = /^(?:tokamak-feature|worker):entry$/;
const inside = (path: string) => path !== ".." && !path.startsWith("../") && !isAbsolute(path);

export function artifactSourcePath(root: string, base: string, source: string): string {
if (!source || syntheticEntry.test(source)) return source;
const path = source.startsWith("file:") ? fileURLToPath(source) : resolve(base, source);
const installed = portable(path);
const dependency = installed.lastIndexOf("/node_modules/");
if (dependency !== -1) return installed.slice(dependency + 1);
const local = portable(relative(root, path));
if (inside(local)) return local;
const workspace = portable(relative(packages, path));
if (inside(workspace)) return `node_modules/@tokamak/${workspace}`;
return `external/${basename(path)}`;
}

export function artifactSourceMap(root: string, file: string, text: string) {
const map = JSON.parse(text) as {
file?: string;
sourceRoot?: string;
sources: string[];
sourcesContent?: (string | null)[];
debugId?: string;
};
// Bun's debug ID includes its original path comments and source map paths.
delete map.debugId;
const base = resolve(dirname(file), map.sourceRoot ?? "");
delete map.sourceRoot;
if (map.file) map.file = basename(map.file);
map.sources = map.sources.map((source, index) => {
const content = map.sourcesContent?.[index];
if (syntheticEntry.test(source) && content && map.sourcesContent) {
// The synthetic entry imports absolute files so the compiler can resolve them.
map.sourcesContent[index] = content.replace(/\bfrom ("(?:[^"\\]|\\.)*")/g, (statement, specifier) => {
const path = JSON.parse(specifier) as string;
return isAbsolute(path) ? `from ${JSON.stringify(artifactSourcePath(root, base, path))}` : statement;
});
}
return artifactSourcePath(root, base, source);
});
return map;
}

export function withoutBunPathComments(code: string) {
const file = ts.createSourceFile("bundle.js", code, ts.ScriptTarget.Latest, false, ts.ScriptKind.JS);
const edits: { pos: number; end: number }[] = [];
for (const node of [...file.statements, file.endOfFileToken]) {
for (const { pos, end } of ts.getLeadingCommentRanges(code, node.pos) ?? []) {
const comment = code.slice(pos, end);
if (/^\/\/ (?:\.{1,2}[/\\]|\/|[a-z]:[/\\]|file:)/i.test(comment) || comment.startsWith("//# debugId="))
edits.push({ pos, end });
}
}
// Leave the line breaks in place so the generated source map remains valid.
for (const { pos, end } of edits.reverse()) code = code.slice(0, pos) + code.slice(end);
return code;
}
2 changes: 2 additions & 0 deletions packages/build/src/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ const config =
? resolve(root, "vite.config.ts")
: false;
async function run() {
// CLI builds give authored Vite hooks their plugin's working directory.
if (mode) process.chdir(root);
let directory: string | undefined;
function verifyOutput(output: string) {
if (mode && output !== resolve(realpathSync(root), "dist") && output !== resolve(root, "dist"))
Expand Down
45 changes: 31 additions & 14 deletions packages/build/src/descriptor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ import ts from "typescript";
import { existsSync, readFileSync, readdirSync } from "node:fs";
import { readFile } from "node:fs/promises";
import { basename, relative, resolve, sep } from "node:path";
import { definePlugin, type PluginDefinition } from "@tokamak/sdk";
import { definePlugin, PluginDefinitionError, type PluginDefinition } from "../../sdk/src/plugin";
import { BuildError, sourceLocation } from "./diagnostics";

export function serverDirectory(root: string): string {
return resolve(root, existsSync(resolve(root, "src/server")) ? "src/server" : "backend");
Expand All @@ -23,8 +24,10 @@ export function readDescriptor(root: string): PluginDefinition | undefined {
const path = resolve(root, "plugin.ts");
if (!existsSync(path)) return;
const source = ts.createSourceFile(path, readFileSync(path, "utf8"), ts.ScriptTarget.Latest, true);
const fail = (): never => {
throw new Error("plugin.ts must export default definePlugin({...}) with static literal metadata");
const at = (node: ts.Node, message: string) =>
new BuildError([{ ...sourceLocation(path, source.text, node.getStart(source)), message }]);
const fail = (node: ts.Node = source): never => {
throw at(node, "plugin.ts must export default definePlugin({...}) with static literal metadata");
};
const helper = source.statements.flatMap((statement) =>
ts.isImportDeclaration(statement) &&
Expand All @@ -38,14 +41,15 @@ export function readDescriptor(root: string): PluginDefinition | undefined {
: [],
);
const assignment = source.statements.find(ts.isExportAssignment);
if (!assignment || assignment.isExportEquals || !ts.isCallExpression(assignment.expression)) return fail();
if (!assignment || assignment.isExportEquals || !ts.isCallExpression(assignment.expression))
return fail(assignment);
const call = assignment.expression;
if (
!ts.isIdentifier(call.expression) ||
!helper.includes(call.expression.text) ||
call.arguments.length !== 1
)
return fail();
return fail(call);
function literal(node: ts.Node): unknown {
if (ts.isStringLiteralLike(node)) return node.text;
if (ts.isArrayLiteralExpression(node)) return node.elements.map(literal);
Expand All @@ -56,16 +60,23 @@ export function readDescriptor(root: string): PluginDefinition | undefined {
!ts.isPropertyAssignment(property) ||
!(ts.isIdentifier(property.name) || ts.isStringLiteral(property.name))
)
return fail();
return fail(property);
const key = property.name.text;
if (Object.hasOwn(result, key)) return fail();
if (Object.hasOwn(result, key)) return fail(property);
result[key] = literal(property.initializer);
}
return result;
}
return fail();
return fail(node);
}
const value = literal(call.arguments[0]!) as Record<string, unknown>;
const descriptor = call.arguments[0]!;
if (!ts.isObjectLiteralExpression(descriptor)) return fail(descriptor);
const value = literal(descriptor) as Record<string, unknown>;
const fields = new Map(
descriptor.properties
.filter(ts.isPropertyAssignment)
.map((property) => [property.name.getText(source).replace(/^['"]|['"]$/g, ""), property]),
);
const allowed = [
"id",
"version",
Expand All @@ -77,24 +88,30 @@ export function readDescriptor(root: string): PluginDefinition | undefined {
"runtime",
"maintenance",
];
if (!value || Array.isArray(value) || Object.keys(value).some((key) => !allowed.includes(key)))
return fail();
const unknown = Object.keys(value).find((key) => !allowed.includes(key));
if (unknown !== undefined) return fail(fields.get(unknown) ?? descriptor);
for (const key of ["depends", "optional"]) {
if (value[key] === undefined) continue;
const ids = value[key];
if (!Array.isArray(ids) || ids.some((id) => typeof id !== "string") || new Set(ids).size !== ids.length)
return fail();
return fail(fields.get(key) ?? descriptor);
value[key] = Object.fromEntries(ids.map((id) => [id, "*"]));
}
return value as unknown as PluginDefinition;
try {
return definePlugin(value as unknown as PluginDefinition);
} catch (error) {
if (!(error instanceof Error)) throw error;
const node = error instanceof PluginDefinitionError ? fields.get(error.field) : undefined;
throw at(node ?? descriptor, error.message);
}
}

export async function readSourcePlugin(
root: string,
prefetched?: Record<string, unknown> | null,
): Promise<PluginDefinition> {
const descriptor = readDescriptor(root);
if (descriptor) return definePlugin(descriptor);
if (descriptor) return descriptor;
const manifest =
prefetched === undefined ? JSON.parse(await readFile(resolve(root, "package.json"), "utf8")) : prefetched;
if (!manifest) throw new Error("Expected plugin.ts or package.json");
Expand Down
9 changes: 8 additions & 1 deletion packages/build/src/ui-build-config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { existsSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { compileCss } from "./css.ts";
import { clientEntry } from "./descriptor.ts";
import { artifactSourcePath } from "./artifact-sources.ts";

import { windowSharedImports } from "./shared-libraries.mts";

Expand All @@ -23,7 +24,13 @@ export function uiBuildConfig(root: string): UserConfig {
fileName: () => "ui.js",
cssFileName: "styles",
},
rollupOptions: { output: { codeSplitting: false } },
rolldownOptions: {
experimental: { attachDebugInfo: "none" },
output: {
codeSplitting: false,
sourcemapPathTransform: (source, map) => artifactSourcePath(root, dirname(map), source),
},
},
},
};
}
Expand Down
41 changes: 25 additions & 16 deletions packages/build/src/vite.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { fileURLToPath } from "node:url";
import { checkPluginImports } from "../../sdk/tools/plugin-boundary.mjs";
import { pluginBoundary } from "./plugin-boundary.ts";
import { artifactBundle } from "./artifact-bundle.ts";
import { artifactSourceMap, withoutBunPathComments } from "./artifact-sources.ts";
import solid from "@solidjs/vite-plugin";
import { clientSurface } from "./client-surface.ts";
import { uiBuildConfig, clientAssets } from "./ui-build-config.ts";
Expand Down Expand Up @@ -126,15 +127,9 @@ export function tokamak(
// in QuickJS. Keep that source authoritative instead of importing user
// schema modules into the build host or duplicating them in manifests.
const bundle = resolve(server, "dist/bundle.js");
const source = await readFile(bundle, "utf8");
const map = JSON.parse(await readFile(`${bundle}.map`, "utf8"));
// SDK maps are relative to the server half’s dist/. Rebase source names when the
// unchanged bundle is packaged into app/dist; retain original contents.
map.sources = map.sources.map((source: string) =>
relative(resolve(root, "dist"), resolve(dirname(bundle), source))
.split(sep)
.join("/"),
);
const source = withoutBunPathComments(await readFile(bundle, "utf8"));
const mapFile = `${bundle}.map`;
const map = artifactSourceMap(root, mapFile, await readFile(mapFile, "utf8"));
this.emitFile({ type: "asset", fileName: "bundle.js", source });
this.emitFile({ type: "asset", fileName: "bundle.js.map", source: JSON.stringify(map) });
// References are source-aware, matching the existing SDK contract.
Expand All @@ -145,13 +140,27 @@ export function tokamak(
);
this.emitFile({ type: "asset", fileName: "api.ts", source: api });
if (plugin.runtime === "worker") {
for (const name of ["worker.js", "worker.js.map", "worker-entry.js"]) {
this.emitFile({
type: "asset",
fileName: name,
source: await readFile(resolve(server, "dist", name), "utf8"),
});
}
const workerMapFile = resolve(server, "dist/worker.js.map");
const workerMap = artifactSourceMap(root, workerMapFile, await readFile(workerMapFile, "utf8"));
const { sourcesContent: _sources, ...runtimeMap } = workerMap;
const worker = await readFile(resolve(server, "dist/worker.js"), "utf8");
const mapStart = worker.lastIndexOf("\nexport const workerSourceMap = ");
if (mapStart < 0) throw new Error("Missing runtime source map for the plugin worker");
this.emitFile({
type: "asset",
fileName: "worker.js",
source: `${withoutBunPathComments(worker.slice(0, mapStart))}\nexport const workerSourceMap = ${JSON.stringify(runtimeMap)};\n`,
});
this.emitFile({
type: "asset",
fileName: "worker.js.map",
source: JSON.stringify(workerMap),
});
this.emitFile({
type: "asset",
fileName: "worker-entry.js",
source: withoutBunPathComments(await readFile(resolve(server, "dist/worker-entry.js"), "utf8")),
});
entries.worker = { entry: "worker-entry.js", code: "worker.js", map: "worker.js.map" };
}
entries.backend = { code: "bundle.js", map: "bundle.js.map", api: "api.ts" };
Expand Down
9 changes: 8 additions & 1 deletion packages/cli/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ Run from a checkout with the pinned Bun:
```sh
proto run bun -- run --silent tokamak --help
proto run bun -- run --silent tokamak plugin build /path/to/my-plugin --json
proto run bun -- run --silent tokamak plugin pack /path/to/my-plugin --json
proto run bun -- run --silent tokamak plugin add /path/to/my-plugin
proto run bun -- run --silent tokamak plugin list --json
proto run bun -- run --silent tokamak plugin logs my-plugin --errors --json
Expand All @@ -15,7 +16,12 @@ proto run bun -- run --silent tokamak report --out diagnostics.md
```

`build` runs the artifact build once, defaults to the current folder, and
doesn't need a server. `add` reads the plugin's static identity from `plugin.ts`
doesn't need a server. `pack` also works without a server and defaults to the
current folder. It builds the plugin and writes `<id>-<version>.zip` in the
folder where the command was run. The ZIP contains `manifest.json` and the
files listed in its hashes. The build id is the SHA-256 of the manifest bytes.

`add` reads the plugin's static identity from `plugin.ts`
(or the build's `package.json` convention), and doesn't build it. `add` and
`remove` edit the index. Run `tokamak plugin reload [id]` to build and apply indexed plugins, or omit the id to also apply removals. Both commands serialize index updates with an exclusive owner
file: a dead owner can be removed, and a live owner is never displaced. An
Expand Down Expand Up @@ -100,6 +106,7 @@ nonzero on failure. The CLI never prompts. Success has `ok: true` and:
| Command | Additional result fields |
| --------------- | ------------------------------------------------------------------ |
| `plugin build` | `id`, `outputFolder` |
| `plugin pack` | `path` (absolute), `id`, `version`, `buildId` |
| `plugin add` | `id`, `folder`, `reloadRequired: true`, `precedence: "index"` |
| `plugin remove` | `id`, `reloadRequired: true` |
| `plugin list` | `plugins`: an array of `{ id, version, fromIndex }` |
Expand Down
5 changes: 4 additions & 1 deletion packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,8 @@
"name": "@tokamak/cli",
"version": "0.1.0",
"private": true,
"type": "module"
"type": "module",
"dependencies": {
"fflate": "0.8.3"
}
}
2 changes: 2 additions & 0 deletions packages/cli/src/commands.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { Command } from "./command";
import build from "./commands/build";
import pack from "./commands/pack";
import add from "./commands/add";
import remove from "./commands/remove";
import list from "./commands/list";
Expand All @@ -17,6 +18,7 @@ const peopleAndServer = [members, invite, revoke, serverStart, serverStop, serve

export const commands: readonly Command[] = [
build,
pack,
add,
remove,
list,
Expand Down
Loading