From d249ca07287f0dac1afaf50a3541db16fe3faea5 Mon Sep 17 00:00:00 2001 From: Johannes Stein Date: Mon, 9 Nov 2026 09:52:10 +0000 Subject: [PATCH] fix(output): replace unpaired surrogates in UTF-8 text --- scripts/test-cli-output-encoding.ts | 79 ++++++++++++++++++++++++ scripts/test-cli.ts | 4 ++ source/shared/ConsoleOutput.Test.pas | 25 ++++++++ source/shared/ProcessEncoding.pas | 89 ++++++++++++++++++---------- 4 files changed, 167 insertions(+), 30 deletions(-) create mode 100644 scripts/test-cli-output-encoding.ts diff --git a/scripts/test-cli-output-encoding.ts b/scripts/test-cli-output-encoding.ts new file mode 100644 index 000000000..6fa3c3aae --- /dev/null +++ b/scripts/test-cli-output-encoding.ts @@ -0,0 +1,79 @@ +#!/usr/bin/env bun +/** Raw UTF-8 output checks: decoding stdout first would hide malformed bytes. */ +import { readFileSync, writeFileSync } from "fs"; +import { join } from "path"; +import { clean, mkdtemp } from "./test-cli/tmpdir"; +import { RUNNER, TESTRUNNER } from "./test-cli/binaries"; + +export function verifyOutputEncoding(runner = RUNNER): void { + const cases = [ + ["empty", '""', ""], + ["NUL", '"a\\u0000b"', "a\0b"], + ["ASCII", '"plain ?"', "plain ?"], + ["valid pair", '"\\uD83D\\uDE00"', "😀"], + ["lead", '"\\uD800"', "�"], + ["trail", '"\\uDC00"', "�"], + ["reversed", '"\\uDC00\\uD800"', "��"], + ["mixed", '"a\\uD800b\\uDC00é\\uD83D\\uDE00"', "a�b�é😀"], + ] as const; + let failures = 0; + for (const [label, expression, expected] of cases) { + for (const [method, prefix] of [["log", ""], ["info", "Info: "], ["debug", "Debug: "], ["warn", "Warning: "], ["error", "Error: "]] as const) { + const result = Bun.spawnSync([runner, "-"], { + stdin: Buffer.from(`console.${method}(${expression});`), + stdout: "pipe", stderr: "pipe", + }); + const bytes = Buffer.from(prefix + expected + (process.platform === "win32" ? "\r\n" : "\n")); + if (result.exitCode !== 0 || !result.stdout.subarray(0, result.stdout.indexOf(10) + 1).equals(bytes) || result.stderr.length !== 0) { + console.error(`${method} ${label}: expected ${bytes.toString("hex")}, got ${result.stdout.toString("hex")} stderr=${result.stderr.toString("hex")}`); + failures++; + } + } + // Only the first line is script output; the runner adds its timing banner. + // JSON capture crosses the RTL's string list before JSON escaping. A host + // callback itself still receives UTF-16; these captured lines are UTF-8 text. + const json = Bun.spawnSync([runner, "--output=json", "-"], { + stdin: Buffer.from(`console.log(${expression});`), stdout: "pipe", stderr: "pipe", + }); + const captured = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(json.stdout)).output; + if (json.exitCode !== 0 || captured.length !== 1 || captured[0] !== expected) { + console.error(`JSON ${label}: ${JSON.stringify(captured)}`); + failures++; + } + // The explicit log-file encoder is an already-passing control. + const tmp = mkdtemp("goccia-output-encoding-"); + try { + const path = join(tmp, "console.log"); + const logged = Bun.spawnSync([runner, `--log=${path}`, "-"], { + stdin: Buffer.from(`console.log(${expression});`), stdout: "pipe", stderr: "pipe", + }); + const expectedLog = Buffer.from(`[log] ${expected}` + (process.platform === "win32" ? "\r\n" : "\n")); + if (logged.exitCode !== 0 || !readFileSync(path).equals(expectedLog)) { + console.error(`log file ${label} differs`); + failures++; + } + } finally { clean(tmp); } + console.log(`checked ${label}`); + } + if (failures) throw new Error(`${failures} output encoding failures`); + console.log("Output encoding checks passed."); +} + +/** Test names and failed-test details cross the reporter's string lists. */ +export function verifyReporterEncoding(binary = TESTRUNNER): void { + const tmp = mkdtemp("goccia-reporter-encoding-"); + try { + const path = join(tmp, "report.js"); + writeFileSync(path, 'test("name\\uD800\\uDC00\\uDC00\\uD800?", () => { expect(1).toBe(2); });'); + const result = Bun.spawnSync([binary, "-P", "--no-progress", path], { stdout: "pipe", stderr: "pipe" }); + if (result.exitCode !== 1 || !result.stdout.includes(Buffer.from("name𐀀��?")) || + result.stdout.includes(Buffer.from("name𐀀???"))) + throw new Error(`Reporter encoding differs: ${result.stdout.toString("hex")}`); + } finally { clean(tmp); } + console.log("Reporter encoding checks passed."); +} + +if (import.meta.main) { + verifyOutputEncoding(process.argv[2] ?? RUNNER); + if (!process.argv[2]) verifyReporterEncoding(); +} diff --git a/scripts/test-cli.ts b/scripts/test-cli.ts index 1f69e9bc7..8bf649094 100644 --- a/scripts/test-cli.ts +++ b/scripts/test-cli.ts @@ -10,6 +10,7 @@ */ import { $ } from "bun"; +import { verifyOutputEncoding, verifyReporterEncoding } from "./test-cli-output-encoding"; import { writeFileSync, readFileSync, @@ -32,6 +33,9 @@ import { import { containsLine, normalizeLineEndings, runLoaderJson } from "./test-cli/assertions"; import { mkdtemp, clean } from "./test-cli/tmpdir"; +verifyOutputEncoding(); +verifyReporterEncoding(); + // -- Stdin smoke (Loader) ------------------------------------------------------ console.log("Stdin smoke..."); diff --git a/source/shared/ConsoleOutput.Test.pas b/source/shared/ConsoleOutput.Test.pas index 83d179168..44e2cd825 100644 --- a/source/shared/ConsoleOutput.Test.pas +++ b/source/shared/ConsoleOutput.Test.pas @@ -28,6 +28,7 @@ TConsoleOutputTests = class(TTestSuite) procedure TestCompleteUTF8LengthHoldsBackASplitCharacter; procedure TestCompleteUTF8LengthIgnoresMalformedTails; procedure TestRedirectedOutputWritesUTF8Bytes; + procedure TestUnpairedSurrogatesAreReplaced; procedure TestConsoleOutputGoesToTheWriter; procedure TestConsoleOutputJoinsCharactersSplitAcrossFlushes; procedure TestConsoleOutputFlushesEachLine; @@ -100,6 +101,8 @@ function CompleteLength(const ABytes: RawByteString): SizeInt; procedure TConsoleOutputTests.SetupTests; begin + Test('Output replaces unpaired surrogates and preserves pairs and NUL', + TestUnpairedSurrogatesAreReplaced); Test('CompleteUTF8Length keeps text that ends on a character boundary', TestCompleteUTF8LengthKeepsWholeCharacters); Test('CompleteUTF8Length holds back a character split at the end', @@ -120,6 +123,28 @@ procedure TConsoleOutputTests.SetupTests; TestConsoleWriterFailureSetsIOResult); end; +procedure TConsoleOutputTests.TestUnpairedSurrogatesAreReplaced; +var + Text: UnicodeString; + Expected: RawByteString; +begin + // Build at run time so FPC cannot fold the conversion of a constant. + Text := Copy('a' + #$D800 + 'b' + #$DC00 + #$DC00#$D800 + + #$D83D#$DE00 + #0 + '?', 1, 11); + Expected := RawBytes([$61, $EF, $BF, $BD, $62, $EF, $BF, $BD, + $EF, $BF, $BD, $EF, $BF, $BD, $F0, $9F, $98, $80, $00, $3F]); + OpenTarget; + UseUnicodeOutput(FTarget, ProbeRedirected, RecordWrite); + Write(FTarget, Text); + Expect(TargetFileBytes = Expected).ToBe(True); + CloseTarget; + OpenTarget; + UseUnicodeOutput(FTarget, ProbeConsole, RecordWrite); + Write(FTarget, Text); + CloseFile(FTarget); + Expect(Written = Expected).ToBe(True); +end; + procedure TConsoleOutputTests.BeforeEach; begin // A failed assertion skips the rest of its test, so the file it left open diff --git a/source/shared/ProcessEncoding.pas b/source/shared/ProcessEncoding.pas index 285e64498..bf54bab91 100644 --- a/source/shared/ProcessEncoding.pas +++ b/source/shared/ProcessEncoding.pas @@ -25,7 +25,9 @@ thread to write. On Linux and macOS this unit installs cwstring, which makes those - conversions through iconv, with a direct path for ASCII text in front of it. } + conversions through iconv, with a direct path for ASCII text in front of it. + UTF-8 output uses TextEncoding on every FPC platform so an unpaired UTF-16 + surrogate becomes U+FFFD, independently of the platform converter. } unit ProcessEncoding; @@ -35,11 +37,21 @@ interface implementation -{$IF DEFINED(MSWINDOWS)} uses +{$IF DEFINED(FPC) AND DEFINED(UNIX)} + cwstring, +{$ENDIF} + SysUtils, +{$IF DEFINED(MSWINDOWS)} Windows, +{$ENDIF} + +{$IF DEFINED(MSWINDOWS)} + ConsoleOutput, +{$ENDIF} + TextEncoding; - ConsoleOutput; +{$IF DEFINED(MSWINDOWS)} const { At most as many UTF-16 code units as UTF-8 bytes: a one-byte to @@ -103,31 +115,18 @@ function WriteConsoleUTF8(const AHandle: THandle; const AText: PAnsiChar; Result := True; end; -initialization - SetMultiByteConversionCodePage(CP_UTF8); - SetMultiByteRTLFileSystemCodePage(CP_UTF8); - // A GUI application's StdOut and StdErr show a message box, and its other - // standard files are not open, so they keep the RTL's settings. - if IsConsole then - begin - UseUnicodeOutput(Output, IsConsoleHandle, WriteConsoleUTF8); - UseUnicodeOutput(StdOut, IsConsoleHandle, WriteConsoleUTF8); - UseUnicodeOutput(ErrOutput, IsConsoleHandle, WriteConsoleUTF8); - UseUnicodeOutput(StdErr, IsConsoleHandle, WriteConsoleUTF8); - end; -{$ELSEIF DEFINED(FPC) AND DEFINED(UNIX)} -uses - cwstring; +{$ENDIF} +{$IFDEF FPC} { cwstring converts between UTF-16 and the system code page through iconv, which costs a converter lookup and a call per string. Most strings that cross the boundary are ASCII — resource strings, identifiers, file names — - and ASCII is the same bytes in UTF-8 and in ASCII, so those are converted - here and everything else is handed to cwstring as before. } + and ASCII is the same bytes in UTF-8 and in ASCII. Non-ASCII UTF-8 uses + the shared replacement encoder; other code pages retain the RTL policy. } var - GIconvUnicode2AnsiMove: procedure(ASource: PUnicodeChar; + GPlatformUnicode2AnsiMove: procedure(ASource: PUnicodeChar; var ADest: RawByteString; ACodePage: TSystemCodePage; ALength: SizeInt); - GIconvAnsi2UnicodeMove: procedure(ASource: PAnsiChar; + GPlatformAnsi2UnicodeMove: procedure(ASource: PAnsiChar; ACodePage: TSystemCodePage; var ADest: UnicodeString; ALength: SizeInt); function IsASCIICompatibleCodePage(const ACodePage: TSystemCodePage): Boolean; @@ -135,11 +134,13 @@ function IsASCIICompatibleCodePage(const ACodePage: TSystemCodePage): Boolean; Result := (ACodePage = CP_UTF8) or (ACodePage = CP_ASCII); end; -procedure ASCIIUnicode2AnsiMove(ASource: PUnicodeChar; +procedure OutputUnicode2AnsiMove(ASource: PUnicodeChar; var ADest: RawByteString; ACodePage: TSystemCodePage; ALength: SizeInt); var Target: PAnsiChar; I: SizeInt; + Text: UnicodeString; + Bytes: TBytes; begin if IsASCIICompatibleCodePage(ACodePage) then begin @@ -156,7 +157,19 @@ procedure ASCIIUnicode2AnsiMove(ASource: PUnicodeChar; Exit; end; end; - GIconvUnicode2AnsiMove(ASource, ADest, ACodePage, ALength); + if ACodePage = CP_UTF8 then + begin + // Preserve the explicit length, including NUL, without invoking another + // RTL conversion from inside its own string-manager callback. + SetString(Text, ASource, ALength); + Bytes := EncodeUTF8WithReplacement(Text); + SetLength(ADest, Length(Bytes)); + if Length(Bytes) > 0 then + Move(Bytes[0], ADest[1], Length(Bytes)); + SetCodePage(ADest, CP_UTF8, False); + Exit; + end; + GPlatformUnicode2AnsiMove(ASource, ADest, ACodePage, ALength); end; procedure ASCIIAnsi2UnicodeMove(ASource: PAnsiChar; ACodePage: TSystemCodePage; @@ -179,19 +192,35 @@ procedure ASCIIAnsi2UnicodeMove(ASource: PAnsiChar; ACodePage: TSystemCodePage; Exit; end; end; - GIconvAnsi2UnicodeMove(ASource, ACodePage, ADest, ALength); + GPlatformAnsi2UnicodeMove(ASource, ACodePage, ADest, ALength); end; -procedure InstallASCIIConversions; +procedure InstallConversions; begin - GIconvUnicode2AnsiMove := WideStringManager.Unicode2AnsiMoveProc; - GIconvAnsi2UnicodeMove := WideStringManager.Ansi2UnicodeMoveProc; - WideStringManager.Unicode2AnsiMoveProc := ASCIIUnicode2AnsiMove; + GPlatformUnicode2AnsiMove := WideStringManager.Unicode2AnsiMoveProc; + GPlatformAnsi2UnicodeMove := WideStringManager.Ansi2UnicodeMoveProc; + WideStringManager.Unicode2AnsiMoveProc := OutputUnicode2AnsiMove; WideStringManager.Ansi2UnicodeMoveProc := ASCIIAnsi2UnicodeMove; end; +{$ENDIF} + initialization - InstallASCIIConversions; +{$IF DEFINED(MSWINDOWS)} + SetMultiByteConversionCodePage(CP_UTF8); + SetMultiByteRTLFileSystemCodePage(CP_UTF8); + // A GUI application's StdOut and StdErr show a message box, and its other + // standard files are not open, so they keep the RTL's settings. + if IsConsole then + begin + UseUnicodeOutput(Output, IsConsoleHandle, WriteConsoleUTF8); + UseUnicodeOutput(StdOut, IsConsoleHandle, WriteConsoleUTF8); + UseUnicodeOutput(ErrOutput, IsConsoleHandle, WriteConsoleUTF8); + UseUnicodeOutput(StdErr, IsConsoleHandle, WriteConsoleUTF8); + end; +{$ENDIF} +{$IFDEF FPC} + InstallConversions; {$ENDIF} end.