Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion e2e/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
},
"lib": ["ES2022", "DOM"],
"skipLibCheck": true,
"allowJs": true
"allowJs": true,
"allowImportingTsExtensions": true
},
"include": [
"run.ts",
Expand Down
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"workspaces": [
"app",
"packages/build",
"packages/cli",
"packages/sdk",
"packages/runtime",
"packages/ui",
Expand All @@ -18,11 +19,12 @@
"e2e": "moon run :e2e",
"app": "moon run app:run",
"app:dev": "moon run app:run",
"tokamak": "bun packages/cli/src/main.ts",
"plugin:build": "proto run bun -- packages/build/src/build.ts",
"server:start": "moon run app:server",
"server:stop": "proto run bun -- packages/runtime/tools/run-app.ts stop debug",
"server:status": "proto run bun -- packages/runtime/tools/run-app.ts status debug",
"typecheck": "moon run sdk:check build:check runtime:ts-check ui:check app:check workspace:plugins-check workspace:e2e-check workspace:wiki-check",
"typecheck": "proto run bun -- check --no-script --project packages/cli/tsconfig.json && moon run sdk:check build:check runtime:ts-check ui:check app:check workspace:plugins-check workspace:e2e-check workspace:wiki-check",
"e2e:browser": "proto run bun -- e2e/browser.ts"
},
"devDependencies": {
Expand Down
121 changes: 73 additions & 48 deletions packages/build/src/build.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
/** Public artifact build: maintained compilers, one deterministic publisher. */
import { build } from "vite";
import { existsSync } from "node:fs";
import { build, resolveConfig, type InlineConfig } from "vite";
import { existsSync, realpathSync } from "node:fs";
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { buildDiagnostics } from "./diagnostics";
import { tokamak } from "./vite";

const args = process.argv.slice(2);
const mode = args[0] === "--result-json" || args[0] === "--check-index" ? args.shift() : undefined;
const report = (result: object) => console.log(`Tokamak build result: ${JSON.stringify(result)}`);
if (
args.length > 2 ||
(args.length === 2 && args[0] !== "--config") ||
Expand All @@ -20,53 +23,75 @@ const config =
: existsSync(resolve(root, "vite.config.ts"))
? resolve(root, "vite.config.ts")
: false;
let directory: string | undefined;
const result = await build({
...(config ? { configFile: config } : { root, configFile: false }),
build: { write: false },
plugins: [
...(!config ? [tokamak()] : []),
{
name: "tokamak-output",
configResolved(config) {
if (config.build.write !== false) throw new Error("Generate all output before publishing");
directory = resolve(config.root, config.build.outDir);
async function run() {
let directory: string | undefined;
function verifyOutput(output: string) {
if (mode && output !== resolve(realpathSync(root), "dist") && output !== resolve(root, "dist"))
throw new Error(`Indexed plugin builds must use ${resolve(root, "dist")} (configured: ${output})`);
}
const options: InlineConfig = {
...(config ? { root, configFile: config } : { root, configFile: false }),
build: { write: false },
plugins: [
...(!config ? [tokamak()] : []),
{
name: "tokamak-output",
configResolved(config) {
if (config.build.write !== false) throw new Error("Generate all output before publishing");
directory = resolve(config.root, config.build.outDir);
verifyOutput(directory);
},
},
},
],
});
if (!directory) throw new Error("Artifact output directory was not resolved");
const files = (Array.isArray(result) ? result : [result]).flatMap((output) => {
if (!("output" in output)) throw new Error("Artifact build does not support watch mode");
return output.output;
});
// Manifest last; the M1 installer still needs its own complete-artifact staging.
files.sort(
(a, b) =>
Number(a.fileName === "manifest.json") - Number(b.fileName === "manifest.json") ||
a.fileName.localeCompare(b.fileName),
);
let changed = 0;
let unchanged = 0;
for (const file of files) {
const bytes = Buffer.from(file.type === "chunk" ? file.code : file.source);
const destination = resolve(directory, file.fileName);
try {
if (bytes.equals(await readFile(destination))) {
unchanged++;
continue;
}
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
],
};
if (mode === "--check-index") {
if (config) await resolveConfig(options, "build", "production", "production");
report({ ok: true });
return;
}
await mkdir(dirname(destination), { recursive: true });
const temporary = `${destination}.${process.pid}.tmp`;
try {
await writeFile(temporary, bytes);
await rename(temporary, destination);
} finally {
await rm(temporary, { force: true });
const result = await build(options);
if (!directory) throw new Error("Artifact output directory was not resolved");
const files = (Array.isArray(result) ? result : [result]).flatMap((output) => {
if (!("output" in output)) throw new Error("Artifact build does not support watch mode");
return output.output;
});
// Manifest last; the M1 installer still needs its own complete-artifact staging.
files.sort(
(a, b) =>
Number(a.fileName === "manifest.json") - Number(b.fileName === "manifest.json") ||
a.fileName.localeCompare(b.fileName),
);
let changed = 0;
let unchanged = 0;
for (const file of files) {
const bytes = Buffer.from(file.type === "chunk" ? file.code : file.source);
const destination = resolve(directory, file.fileName);
try {
if (bytes.equals(await readFile(destination))) {
unchanged++;
continue;
}
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
}
await mkdir(dirname(destination), { recursive: true });
const temporary = `${destination}.${process.pid}.tmp`;
try {
await writeFile(temporary, bytes);
await rename(temporary, destination);
} finally {
await rm(temporary, { force: true });
}
changed++;
}
changed++;
if (mode) report({ ok: true, outputFolder: directory });
else console.log(`Plugin artifact: ${changed} changed, ${unchanged} unchanged`);
}

try {
await run();
} catch (error) {
if (!mode) throw error;
report({ ok: false, diagnostics: buildDiagnostics(error, root) });
process.exitCode = 1;
}
console.log(`Plugin artifact: ${changed} changed, ${unchanged} unchanged`);
10 changes: 2 additions & 8 deletions packages/build/src/contributions.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { dirname } from "node:path";
import { pluginSlots, readPluginCompilerOptions } from "../../sdk/tools/plugin-boundary.mjs";
import { clientEntry } from "./descriptor.ts";
import { BuildError, typescriptDiagnostics } from "./diagnostics.ts";
/** Use the compiler's export/alias graph, as backend exports do. Keep TS 5.9.3:
* 6.0.3 is the newest JS API release; 7.x has no JS API. Upgrading belongs to
* a separate tooling change, not the plugin wire-format implementation. */
Expand Down Expand Up @@ -36,14 +37,7 @@ export function uiContributions(root: string) {
...program.getGlobalDiagnostics(),
...program.getSemanticDiagnostics(),
];
if (diagnostics.length)
throw new Error(
ts.formatDiagnostics(diagnostics, {
getCanonicalFileName: (file) => file,
getCurrentDirectory: () => root,
getNewLine: () => "\n",
}),
);
if (diagnostics.length) throw new BuildError(typescriptDiagnostics(diagnostics, root));
const checker = program.getTypeChecker();
const source = program.getSourceFile(entry);
if (!source) throw new Error(`Missing plugin UI entry ${entry}`);
Expand Down
19 changes: 18 additions & 1 deletion packages/build/src/descriptor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@ import { readPluginCompilerOptions } from "../../sdk/tools/plugin-boundary.mjs";
import { isServerModule } from "../../sdk/tools/server-files.mjs";
import ts from "typescript";
import { existsSync, readFileSync, readdirSync } from "node:fs";
import { readFile } from "node:fs/promises";
import { basename, relative, resolve, sep } from "node:path";
import type { PluginDefinition } from "@tokamak/sdk";
import { definePlugin, type PluginDefinition } from "@tokamak/sdk";

export function serverDirectory(root: string): string {
return resolve(root, existsSync(resolve(root, "src/server")) ? "src/server" : "backend");
Expand Down Expand Up @@ -88,6 +89,22 @@ export function readDescriptor(root: string): PluginDefinition | undefined {
return value as unknown as PluginDefinition;
}

export async function readSourcePlugin(
root: string,
prefetched?: Record<string, unknown> | null,
): Promise<PluginDefinition> {
const descriptor = readDescriptor(root);
if (descriptor) return definePlugin(descriptor);
const manifest =
prefetched === undefined ? JSON.parse(await readFile(resolve(root, "package.json"), "utf8")) : prefetched;
if (!manifest) throw new Error("Expected plugin.ts or package.json");
if (typeof manifest.name !== "string") throw new Error("Plugin package.json must name the plugin");
return definePlugin({
id: manifest.name.replace(/^@/, "").replaceAll("/", "."),
version: manifest.version ?? "0.1.0",
});
}

/** The existing backend protocol has three public actions. Derive its provider
* from those exports; derive consumers from literal ctx.providers calls. */
export function deriveInterfaces(root: string) {
Expand Down
139 changes: 139 additions & 0 deletions packages/build/src/diagnostics.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import ts from "typescript";

export interface BuildDiagnostic {
message: string;
file?: string;
line?: number;
column?: number;
code?: string;
}

export class BuildError extends Error {
constructor(
readonly diagnostics: BuildDiagnostic[],
options?: ErrorOptions,
) {
super(
diagnostics
.map(
({ file, line, column, code, message }) =>
`${file ? `${file}(${line},${column}): ` : ""}${code ? `error ${code}: ` : ""}${message}`,
)
.join("\n"),
options,
);
}
}

export const sdkDiagnosticPrefix = "Tokamak SDK diagnostics: ";

export function sourceLocation(file: string, text: string, offset: number) {
const lines = text.slice(0, offset).split(/\r\n|\n|\r/);
return { file: resolve(file), line: lines.length, column: lines.at(-1)!.length + 1 };
}

export function locateImportError(error: unknown, importer: string, specifier: string) {
if (!(error instanceof BuildError)) return error;
const file = importer.split("?")[0]!;
let loc = { file: resolve(file), line: 1, column: 1 };
try {
const text = readFileSync(file, "utf8");
const imported = ts
.preProcessFile(text, true, true)
.importedFiles.find(({ fileName }) => fileName === specifier);
if (imported) loc = sourceLocation(file, text, imported.pos);
} catch {
// A virtual or generated importer may not exist on disk.
}
return new BuildError(error.diagnostics.map((diagnostic) => ({ ...loc, ...diagnostic })));
}

export function typescriptDiagnostics(
diagnostics: readonly ts.Diagnostic[],
root: string,
): BuildDiagnostic[] {
function message(text: string | ts.DiagnosticMessageChain, indent = ""): string {
return typeof text === "string"
? text
: [
text.messageText,
...(text.next ?? []).map((child) => indent + " " + message(child, indent + " ")),
].join("\n");
}
return diagnostics.map((diagnostic) => {
const result = { code: `TS${diagnostic.code}`, message: message(diagnostic.messageText) };
if (!diagnostic.file || diagnostic.start === undefined) return result;
const { line, character } = diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start);
return {
...result,
file: resolve(root, diagnostic.file.fileName),
line: line + 1,
column: character + 1,
};
});
}

const xmlEntities: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'" };
const unescapeXml = (value: string) =>
value.replace(/&(amp|lt|gt|quot|apos);/g, (_, key) => xmlEntities[key]!);

export function checkerDiagnostics(output: string, cwd: string): BuildDiagnostic[] {
const record = output
.split("\n")
.reverse()
.find((line) => line.startsWith(sdkDiagnosticPrefix));
if (record) {
const diagnostics: unknown = JSON.parse(record.slice(sdkDiagnosticPrefix.length));
if (Array.isArray(diagnostics)) return diagnostics;
}
const tagged = /<error\b([^>]*)>([\s\S]*?)<\/error>/g;
const blocks = [...output.matchAll(tagged)];
if (blocks.length) {
return blocks.map(([, attributes, text]) => {
const fields = Object.fromEntries(
[...attributes!.matchAll(/(\w+)="([^"]*)"/g)].map(([, key, value]) => [key, unescapeXml(value!)]),
);
return {
...location(cwd, fields.file, Number(fields.line), Number(fields.column)),
code: fields.code,
message: unescapeXml(text!.split(/^<source>\r?$/m, 1)[0]!.trim()),
};
});
}
const plain = /^(?:(.+)\((\d+),(\d+)\): )?error (TS\d+): (.*(?:\n[ \t]+[^\n]+)*)$/gm;
return [...output.matchAll(plain)].map(([, file, line, column, code, message]) => ({
...location(cwd, file, Number(line), Number(column)),
code,
message: message!.trimEnd(),
}));
}

function location(root: string, file?: string, line?: number, column?: number) {
return file && line && Number.isInteger(line)
? { file: resolve(root, file), line, ...(column && Number.isInteger(column) ? { column } : {}) }
: {};
}

export function buildDiagnostics(error: unknown, root: string): BuildDiagnostic[] {
if (!(error instanceof Error)) return [{ message: String(error) }];
const detail = error as Error & {
diagnostics?: BuildDiagnostic[];
errors?: unknown[];
loc?: { file?: string; line?: number; column?: number };
id?: string;
code?: string;
};
if (detail.diagnostics?.length) return detail.diagnostics;
if (Array.isArray(detail.errors) && detail.errors.length)
return detail.errors.flatMap((child) => buildDiagnostics(child, root));
if (detail.cause) return buildDiagnostics(detail.cause, root);
return [
{
...location(root, detail.loc?.file ?? detail.id, detail.loc?.line, detail.loc?.column),
...(typeof detail.code === "string" ? { code: detail.code } : {}),
message: detail.message,
},
];
}
Loading