fix(output): replace unpaired surrogates in UTF-8 text

scripts/test-cli-output-encoding.ts

@@ -0,0 +1,79 @@
+#!/usr/bin/env bun
+/** Raw UTF-8 output checks: decoding stdout first would hide malformed bytes. */
+import { readFileSync, writeFileSync } from "fs";
+import { join } from "path";
+import { clean, mkdtemp } from "./test-cli/tmpdir";
+import { RUNNER, TESTRUNNER } from "./test-cli/binaries";
+
+export function verifyOutputEncoding(runner = RUNNER): void {
+  const cases = [
+    ["empty", '""', ""],
+    ["NUL", '"a\\u0000b"', "a\0b"],
+    ["ASCII", '"plain ?"', "plain ?"],
+    ["valid pair", '"\\uD83D\\uDE00"', "😀"],
+    ["lead", '"\\uD800"', "�"],
+    ["trail", '"\\uDC00"', "�"],
+    ["reversed", '"\\uDC00\\uD800"', "��"],
+    ["mixed", '"a\\uD800b\\uDC00é\\uD83D\\uDE00"', "a�b�é😀"],
+  ] as const;
+  let failures = 0;
+  for (const [label, expression, expected] of cases) {
+    for (const [method, prefix] of [["log", ""], ["info", "Info: "], ["debug", "Debug: "], ["warn", "Warning: "], ["error", "Error: "]] as const) {
+      const result = Bun.spawnSync([runner, "-"], {
+        stdin: Buffer.from(`console.${method}(${expression});`),
+        stdout: "pipe", stderr: "pipe",
+      });
+      const bytes = Buffer.from(prefix + expected + (process.platform === "win32" ? "\r\n" : "\n"));
+      if (result.exitCode !== 0 || !result.stdout.subarray(0, result.stdout.indexOf(10) + 1).equals(bytes) || result.stderr.length !== 0) {
+        console.error(`${method} ${label}: expected ${bytes.toString("hex")}, got ${result.stdout.toString("hex")} stderr=${result.stderr.toString("hex")}`);
+        failures++;
+      }
+    }
+    // Only the first line is script output; the runner adds its timing banner.
+    // JSON capture crosses the RTL's string list before JSON escaping. A host
+    // callback itself still receives UTF-16; these captured lines are UTF-8 text.
+    const json = Bun.spawnSync([runner, "--output=json", "-"], {
+      stdin: Buffer.from(`console.log(${expression});`), stdout: "pipe", stderr: "pipe",
+    });
+    const captured = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(json.stdout)).output;
+    if (json.exitCode !== 0 || captured.length !== 1 || captured[0] !== expected) {
+      console.error(`JSON ${label}: ${JSON.stringify(captured)}`);
+      failures++;
+    }
+    // The explicit log-file encoder is an already-passing control.
+    const tmp = mkdtemp("goccia-output-encoding-");
+    try {
+      const path = join(tmp, "console.log");
+      const logged = Bun.spawnSync([runner, `--log=${path}`, "-"], {
+        stdin: Buffer.from(`console.log(${expression});`), stdout: "pipe", stderr: "pipe",
+      });
+      const expectedLog = Buffer.from(`[log] ${expected}` + (process.platform === "win32" ? "\r\n" : "\n"));
+      if (logged.exitCode !== 0 || !readFileSync(path).equals(expectedLog)) {
+        console.error(`log file ${label} differs`);
+        failures++;
+      }
+    } finally { clean(tmp); }
+    console.log(`checked ${label}`);
+  }
+  if (failures) throw new Error(`${failures} output encoding failures`);
+  console.log("Output encoding checks passed.");
+}
+
+/** Test names and failed-test details cross the reporter's string lists. */
+export function verifyReporterEncoding(binary = TESTRUNNER): void {
+  const tmp = mkdtemp("goccia-reporter-encoding-");
+  try {
+    const path = join(tmp, "report.js");
+    writeFileSync(path, 'test("name\\uD800\\uDC00\\uDC00\\uD800?", () => { expect(1).toBe(2); });');
+    const result = Bun.spawnSync([binary, "-P", "--no-progress", path], { stdout: "pipe", stderr: "pipe" });
+    if (result.exitCode !== 1 || !result.stdout.includes(Buffer.from("name𐀀��?")) ||
+        result.stdout.includes(Buffer.from("name𐀀???")))
+      throw new Error(`Reporter encoding differs: ${result.stdout.toString("hex")}`);
+  } finally { clean(tmp); }
+  console.log("Reporter encoding checks passed.");
+}
+
+if (import.meta.main) {
+  verifyOutputEncoding(process.argv[2] ?? RUNNER);
+  if (!process.argv[2]) verifyReporterEncoding();
+}

scripts/test-cli.ts

@@ -10,6 +10,7 @@
  */
 
 import { $ } from "bun";
+import { verifyOutputEncoding, verifyReporterEncoding } from "./test-cli-output-encoding";
 import {
   writeFileSync,
   readFileSync,
@@ -32,6 +33,9 @@ import {
 import { containsLine, normalizeLineEndings, runLoaderJson } from "./test-cli/assertions";
 import { mkdtemp, clean } from "./test-cli/tmpdir";
 
+verifyOutputEncoding();
+verifyReporterEncoding();
+
 // -- Stdin smoke (Loader) ------------------------------------------------------
 
 console.log("Stdin smoke...");

source/shared/ConsoleOutput.Test.pas

@@ -28,6 +28,7 @@   TConsoleOutputTests = class(TTestSuite)
     procedure TestCompleteUTF8LengthHoldsBackASplitCharacter;
     procedure TestCompleteUTF8LengthIgnoresMalformedTails;
     procedure TestRedirectedOutputWritesUTF8Bytes;
+    procedure TestUnpairedSurrogatesAreReplaced;
     procedure TestConsoleOutputGoesToTheWriter;
     procedure TestConsoleOutputJoinsCharactersSplitAcrossFlushes;
     procedure TestConsoleOutputFlushesEachLine;
@@ -100,6 +101,8 @@ function CompleteLength(const ABytes: RawByteString): SizeInt;
 
 procedure TConsoleOutputTests.SetupTests;
 begin
+  Test('Output replaces unpaired surrogates and preserves pairs and NUL',
+    TestUnpairedSurrogatesAreReplaced);
   Test('CompleteUTF8Length keeps text that ends on a character boundary',
     TestCompleteUTF8LengthKeepsWholeCharacters);
   Test('CompleteUTF8Length holds back a character split at the end',
@@ -120,6 +123,28 @@ procedure TConsoleOutputTests.SetupTests;
     TestConsoleWriterFailureSetsIOResult);
 end;
 
+procedure TConsoleOutputTests.TestUnpairedSurrogatesAreReplaced;
+var
+  Text: UnicodeString;
+  Expected: RawByteString;
+begin
+  // Build at run time so FPC cannot fold the conversion of a constant.
+  Text := Copy('a' + #$D800 + 'b' + #$DC00 + #$DC00#$D800 +
+    #$D83D#$DE00 + #0 + '?', 1, 11);
+  Expected := RawBytes([$61, $EF, $BF, $BD, $62, $EF, $BF, $BD,
+    $EF, $BF, $BD, $EF, $BF, $BD, $F0, $9F, $98, $80, $00, $3F]);
+  OpenTarget;
+  UseUnicodeOutput(FTarget, ProbeRedirected, RecordWrite);
+  Write(FTarget, Text);
+  Expect<Boolean>(TargetFileBytes = Expected).ToBe(True);
+  CloseTarget;
+  OpenTarget;
+  UseUnicodeOutput(FTarget, ProbeConsole, RecordWrite);
+  Write(FTarget, Text);
+  CloseFile(FTarget);
+  Expect<Boolean>(Written = Expected).ToBe(True);
+end;
+
 procedure TConsoleOutputTests.BeforeEach;
 begin
   // A failed assertion skips the rest of its test, so the file it left open

source/shared/ProcessEncoding.pas

@@ -25,7 +25,9 @@
   thread to write.
 
   On Linux and macOS this unit installs cwstring, which makes those
-  conversions through iconv, with a direct path for ASCII text in front of it. }
+  conversions through iconv, with a direct path for ASCII text in front of it.
+  UTF-8 output uses TextEncoding on every FPC platform so an unpaired UTF-16
+  surrogate becomes U+FFFD, independently of the platform converter. }
 
 unit ProcessEncoding;
 
@@ -35,11 +37,21 @@ interface
 
 implementation
 
-{$IF DEFINED(MSWINDOWS)}
 uses
+{$IF DEFINED(FPC) AND DEFINED(UNIX)}
+  cwstring,
+{$ENDIF}
+  SysUtils,
+{$IF DEFINED(MSWINDOWS)}
   Windows,
+{$ENDIF}
+
+{$IF DEFINED(MSWINDOWS)}
+  ConsoleOutput,
+{$ENDIF}
+  TextEncoding;
 
-  ConsoleOutput;
+{$IF DEFINED(MSWINDOWS)}
 
 const
   { At most as many UTF-16 code units as UTF-8 bytes: a one-byte to
@@ -103,43 +115,32 @@ function WriteConsoleUTF8(const AHandle: THandle; const AText: PAnsiChar;
   Result := True;
 end;
 
-initialization
-  SetMultiByteConversionCodePage(CP_UTF8);
-  SetMultiByteRTLFileSystemCodePage(CP_UTF8);
-  // A GUI application's StdOut and StdErr show a message box, and its other
-  // standard files are not open, so they keep the RTL's settings.
-  if IsConsole then
-  begin
-    UseUnicodeOutput(Output, IsConsoleHandle, WriteConsoleUTF8);
-    UseUnicodeOutput(StdOut, IsConsoleHandle, WriteConsoleUTF8);
-    UseUnicodeOutput(ErrOutput, IsConsoleHandle, WriteConsoleUTF8);
-    UseUnicodeOutput(StdErr, IsConsoleHandle, WriteConsoleUTF8);
-  end;
-{$ELSEIF DEFINED(FPC) AND DEFINED(UNIX)}
-uses
-  cwstring;
+{$ENDIF}
 
+{$IFDEF FPC}
 { cwstring converts between UTF-16 and the system code page through iconv,
   which costs a converter lookup and a call per string. Most strings that
   cross the boundary are ASCII — resource strings, identifiers, file names —
-  and ASCII is the same bytes in UTF-8 and in ASCII, so those are converted
-  here and everything else is handed to cwstring as before. }
+  and ASCII is the same bytes in UTF-8 and in ASCII. Non-ASCII UTF-8 uses
+  the shared replacement encoder; other code pages retain the RTL policy. }
 var
-  GIconvUnicode2AnsiMove: procedure(ASource: PUnicodeChar;
+  GPlatformUnicode2AnsiMove: procedure(ASource: PUnicodeChar;
     var ADest: RawByteString; ACodePage: TSystemCodePage; ALength: SizeInt);
-  GIconvAnsi2UnicodeMove: procedure(ASource: PAnsiChar;
+  GPlatformAnsi2UnicodeMove: procedure(ASource: PAnsiChar;
     ACodePage: TSystemCodePage; var ADest: UnicodeString; ALength: SizeInt);
 
 function IsASCIICompatibleCodePage(const ACodePage: TSystemCodePage): Boolean;
 begin
   Result := (ACodePage = CP_UTF8) or (ACodePage = CP_ASCII);
 end;
 
-procedure ASCIIUnicode2AnsiMove(ASource: PUnicodeChar;
+procedure OutputUnicode2AnsiMove(ASource: PUnicodeChar;
   var ADest: RawByteString; ACodePage: TSystemCodePage; ALength: SizeInt);
 var
   Target: PAnsiChar;
   I: SizeInt;
+  Text: UnicodeString;
+  Bytes: TBytes;
 begin
   if IsASCIICompatibleCodePage(ACodePage) then
   begin
@@ -156,7 +157,19 @@ procedure ASCIIUnicode2AnsiMove(ASource: PUnicodeChar;
       Exit;
     end;
   end;
-  GIconvUnicode2AnsiMove(ASource, ADest, ACodePage, ALength);
+  if ACodePage = CP_UTF8 then
+  begin
+    // Preserve the explicit length, including NUL, without invoking another
+    // RTL conversion from inside its own string-manager callback.
+    SetString(Text, ASource, ALength);
+    Bytes := EncodeUTF8WithReplacement(Text);
+    SetLength(ADest, Length(Bytes));
+    if Length(Bytes) > 0 then
+      Move(Bytes[0], ADest[1], Length(Bytes));
+    SetCodePage(ADest, CP_UTF8, False);
+    Exit;
+  end;
+  GPlatformUnicode2AnsiMove(ASource, ADest, ACodePage, ALength);
 end;
 
 procedure ASCIIAnsi2UnicodeMove(ASource: PAnsiChar; ACodePage: TSystemCodePage;
@@ -179,19 +192,35 @@ procedure ASCIIAnsi2UnicodeMove(ASource: PAnsiChar; ACodePage: TSystemCodePage;
       Exit;
     end;
   end;
-  GIconvAnsi2UnicodeMove(ASource, ACodePage, ADest, ALength);
+  GPlatformAnsi2UnicodeMove(ASource, ACodePage, ADest, ALength);
 end;
 
-procedure InstallASCIIConversions;
+procedure InstallConversions;
 begin
-  GIconvUnicode2AnsiMove := WideStringManager.Unicode2AnsiMoveProc;
-  GIconvAnsi2UnicodeMove := WideStringManager.Ansi2UnicodeMoveProc;
-  WideStringManager.Unicode2AnsiMoveProc := ASCIIUnicode2AnsiMove;
+  GPlatformUnicode2AnsiMove := WideStringManager.Unicode2AnsiMoveProc;
+  GPlatformAnsi2UnicodeMove := WideStringManager.Ansi2UnicodeMoveProc;
+  WideStringManager.Unicode2AnsiMoveProc := OutputUnicode2AnsiMove;
   WideStringManager.Ansi2UnicodeMoveProc := ASCIIAnsi2UnicodeMove;
 end;
 
+{$ENDIF}
+
 initialization
-  InstallASCIIConversions;
+{$IF DEFINED(MSWINDOWS)}
+  SetMultiByteConversionCodePage(CP_UTF8);
+  SetMultiByteRTLFileSystemCodePage(CP_UTF8);
+  // A GUI application's StdOut and StdErr show a message box, and its other
+  // standard files are not open, so they keep the RTL's settings.
+  if IsConsole then
+  begin
+    UseUnicodeOutput(Output, IsConsoleHandle, WriteConsoleUTF8);
+    UseUnicodeOutput(StdOut, IsConsoleHandle, WriteConsoleUTF8);
+    UseUnicodeOutput(ErrOutput, IsConsoleHandle, WriteConsoleUTF8);
+    UseUnicodeOutput(StdErr, IsConsoleHandle, WriteConsoleUTF8);
+  end;
+{$ENDIF}
+{$IFDEF FPC}
+  InstallConversions;
 {$ENDIF}
 
 end.