QVAC-27970 feat[api]: expose the tts-ggml 0.15 backend, Core ML, sampling and replay controls

packages/cli/src/serve/extensions/openai/audio.ts

@@ -95,14 +95,19 @@ export function pcmContentType(sampleRate: number): string {
   return `audio/L16; rate=${sampleRate}; channels=1`
 }
 
-// Engine → native sample rate. Mirrors the constants used in the SDK
-// examples (packages/sdk/examples/tts/{chatterbox,supertonic}.ts).
-// TODO(QVAC-18522): add the GGML engine key here when the TTS-GGML migration
-// lands, otherwise the engine falls through to DEFAULT_SAMPLE_RATE and audio
-// plays back at the wrong speed.
+// Engine → native sample rate, the fallback for a run that produced no audio
+// (the route prefers the rate the engine reports). One row per @qvac/tts-ggml
+// engine that resamples; MOSS-SoundEffect emits 48 kHz, the other MOSS
+// engines their native 24 kHz.
 const ENGINE_SAMPLE_RATE: Record<string, number> = {
   chatterbox: 24000,
-  supertonic: 44100
+  supertonic: 44100,
+  parler: 44100,
+  cosyvoice3: 24000,
+  audio8: 44100,
+  moss: 24000,
+  'moss-sfx': 48000,
+  'moss-speech': 24000
 }
 
 // 24 kHz matches OpenAI's documented pcm output (mono, 16-bit signed LE)

packages/cli/test/unit/extensions/openai/audio.test.ts

@@ -118,6 +118,19 @@ describe('resolveSampleRate', () => {
     assert.equal(resolveSampleRate({ ttsEngine: 'supertonic' }), 44100)
   })
 
+  it('knows the native rate of every @qvac/tts-ggml engine', () => {
+    for (const [ttsEngine, sampleRate] of [
+      ['parler', 44100],
+      ['cosyvoice3', 24000],
+      ['audio8', 44100],
+      ['moss', 24000],
+      ['moss-sfx', 48000],
+      ['moss-speech', 24000]
+    ] as const) {
+      assert.equal(resolveSampleRate({ ttsEngine }), sampleRate, ttsEngine)
+    }
+  })
+
   it('is case-insensitive on engine name', () => {
     assert.equal(resolveSampleRate({ ttsEngine: 'CHATTERBOX' }), 24000)
   })

packages/inference/src/api/text-to-speech.ts

@@ -285,6 +285,7 @@ export type TtsSideChannel = {
   sampleRate: Promise<number | undefined>
   stats: Promise<TtsStats | undefined>
   stopReason: Promise<'completed' | 'cancelled' | undefined>
+  speechTokenIds: Promise<number[] | undefined>
   observe(response: TtsResponse): void
   settle(): void
 }
@@ -299,6 +300,7 @@ export function createTtsSideChannel(): TtsSideChannel {
   let resolveSampleRate!: (value: number | undefined) => void
   let resolveStats!: (value: TtsStats | undefined) => void
   let resolveStopReason!: (value: 'completed' | 'cancelled' | undefined) => void
+  let resolveSpeechTokenIds!: (value: number[] | undefined) => void
   const sampleRate = new Promise<number | undefined>((resolve) => {
     resolveSampleRate = resolve
   })
@@ -308,17 +310,24 @@ export function createTtsSideChannel(): TtsSideChannel {
   const stopReason = new Promise<'completed' | 'cancelled' | undefined>((resolve) => {
     resolveStopReason = resolve
   })
+  const speechTokenIds = new Promise<number[] | undefined>((resolve) => {
+    resolveSpeechTokenIds = resolve
+  })
 
   let seenSampleRate: number | undefined
   let latestStats: TtsStats | undefined
   let seenStopReason: 'completed' | 'cancelled' | undefined
+  // CosyVoice3 hands back one array per native synthesis; the chunked paths
+  // run one per sentence, so the run's trajectory is their concatenation.
+  let seenSpeechTokenIds: number[] | undefined
   let settled = false
 
   return {
     text,
     sampleRate,
     stats,
     stopReason,
+    speechTokenIds,
     observe(response) {
       if (response.text !== undefined) seenText = response.text
       // Resolve on the FIRST frame that carries a rate, not at end of stream:
@@ -331,6 +340,12 @@ export function createTtsSideChannel(): TtsSideChannel {
       }
       if (response.stats !== undefined) latestStats = response.stats
       if (response.stopReason !== undefined) seenStopReason = response.stopReason
+      if (response.speechTokenIds !== undefined) {
+        // Append into one owned array: a spread per frame would copy every
+        // token collected so far on each sentence of a long stream.
+        seenSpeechTokenIds ??= []
+        for (const id of response.speechTokenIds) seenSpeechTokenIds.push(id)
+      }
     },
     settle() {
       if (settled) return
@@ -339,6 +354,7 @@ export function createTtsSideChannel(): TtsSideChannel {
       resolveSampleRate(seenSampleRate)
       resolveStats(latestStats)
       resolveStopReason(seenStopReason)
+      resolveSpeechTokenIds(seenSpeechTokenIds)
     }
   }
 }
@@ -389,7 +405,8 @@ function sentenceStreamTts(
     text: side.text,
     sampleRate: side.sampleRate,
     stats: side.stats,
-    stopReason: side.stopReason
+    stopReason: side.stopReason,
+    speechTokenIds: side.speechTokenIds
   }
 }
 
@@ -427,7 +444,8 @@ async function* sentenceChunkUpdates(
     const hasAudio = m.buffer.length > 0
     const hasMeta =
       m.chunkIndex !== undefined ||
-      (typeof m.sentenceChunk === 'string' && m.sentenceChunk.length > 0)
+      (typeof m.sentenceChunk === 'string' && m.sentenceChunk.length > 0) ||
+      m.speechTokenIds !== undefined
     if (hasAudio || hasMeta) {
       yield {
         buffer: hasAudio ? [...m.buffer] : [],
@@ -436,7 +454,8 @@ async function* sentenceChunkUpdates(
         ...(typeof m.sentenceChunk === 'string' && m.sentenceChunk.length > 0
           ? { sentenceChunk: m.sentenceChunk }
           : {}),
-        ...(m.isLast !== undefined ? { isLast: m.isLast } : {})
+        ...(m.isLast !== undefined ? { isLast: m.isLast } : {}),
+        ...(m.speechTokenIds !== undefined ? { speechTokenIds: m.speechTokenIds } : {})
       }
     }
     if (m.done) break
@@ -467,7 +486,8 @@ function plainStreamTts(
     text: side.text,
     sampleRate: side.sampleRate,
     stats: side.stats,
-    stopReason: side.stopReason
+    stopReason: side.stopReason,
+    speechTokenIds: side.speechTokenIds
   }
 }
 
@@ -534,7 +554,8 @@ function collectTts(
     text: side.text,
     sampleRate: side.sampleRate,
     stats: side.stats,
-    stopReason: side.stopReason
+    stopReason: side.stopReason,
+    speechTokenIds: side.speechTokenIds
   }
 }
 

packages/inference/src/plugins/builtin/tts-ggml/fit.ts

@@ -8,6 +8,7 @@ import type { FitLoad, FitRequestPlan } from '@/schemas/plugin'
 
 interface TtsCommon {
   nGpuLayers: number
+  backend?: string
   backendsDir?: string
 }
 
@@ -67,26 +68,36 @@ export function ttsFitRequest(load: FitLoad): FitRequestPlan {
     return { unknown: 'the LavaSR stages have no fitter' }
   }
 
+  // The explicit backend selector resolves the same device the load would, so
+  // the projection lands on that device rather than on the `useGPU` policy's.
   const common: TtsCommon = {
     nGpuLayers: gpuLayersFromCount(config.useGPU, config.nGpuLayers),
+    ...(config.backend !== undefined && { backend: config.backend }),
     ...(config.backendsDir !== undefined && { backendsDir: config.backendsDir })
   }
   const request = (fit: TtsFitRequest): FitRequestPlan => ({ request: fit })
 
   switch (config.ttsEngine) {
     case 'supertonic':
+      // Precision and the f16 weight override change the weight footprint the
+      // fitter sizes, so they travel with the request.
       return request({
         engineType: 'supertonic',
         supertonicModelPath: load.modelPath,
         ...common,
+        ...(config.precision !== undefined && { precision: config.precision }),
+        ...(config.f16Weights !== undefined && { f16Weights: config.f16Weights }),
         ...(config.ttsNumInferenceSteps !== undefined && { steps: config.ttsNumInferenceSteps })
       })
 
     case 'parler':
+      // The decoder sidecar policy decides whether the decoder weights load on
+      // ggml at all; the fitter matches the load's choice.
       return request({
         engineType: 'parler',
         parlerModelPath: load.modelPath,
         ...common,
+        ...(config.coremlDecoder !== undefined && { coremlDecoder: config.coremlDecoder }),
         ...(config.maxFrames !== undefined && { maxFrames: config.maxFrames })
       })
 
@@ -133,7 +144,11 @@ export function ttsFitRequest(load: FitLoad): FitRequestPlan {
         ...(config.streamChunkTokens !== undefined && {
           streamChunkTokens: config.streamChunkTokens
         }),
-        ...(config.durationTokens !== undefined && { durationTokens: config.durationTokens })
+        ...(config.durationTokens !== undefined && { durationTokens: config.durationTokens }),
+        // The context and generation budgets size the KV cache and the
+        // generation buffers the fitter projects.
+        ...(config.nCtx !== undefined && { nCtx: config.nCtx }),
+        ...(config.maxNewTokens !== undefined && { maxNewTokens: config.maxNewTokens })
       })
     }
 

packages/inference/src/plugins/builtin/tts-ggml/ops/text-to-speech-stream.ts

@@ -9,6 +9,7 @@ import type { TtsStats as AddonTtsStats } from '@/utils/addon-responses'
 import {
   type TtsStreamChunk,
   type TtsOpYield,
+  carriesSpeechTokenIds,
   collectTtsStats,
   pcmFrame,
   chunkMetadata
@@ -196,7 +197,9 @@ export async function* textToSpeechStream(
       if (data.outputArray == null) {
         continue
       }
-      if (data.outputArray.length === 0) {
+      // An empty chunk is a native end marker, unless it carries the sentence's
+      // speech tokens (CosyVoice3 native streaming), which the caller wants.
+      if (data.outputArray.length === 0 && !carriesSpeechTokenIds(data)) {
         continue
       }
       yield { ...pcmFrame(data.outputArray), ...chunkMetadata(data) }

packages/inference/src/plugins/builtin/tts-ggml/ops/text-to-speech.ts

@@ -9,6 +9,7 @@ import { PluginRequestValidationFailedError, TextToSpeechFailedError } from '@/e
 import {
   type TtsStreamChunk,
   type TtsOpYield,
+  carriesSpeechTokenIds,
   collectTtsStats,
   chunkMetadata,
   concatPcm,
@@ -144,6 +145,8 @@ export async function* textToSpeech(params: TtsRequest): AsyncGenerator<TtsOpYie
     if (!stream) {
       const parts: ArrayLike<number>[] = []
       let sampleRate: number | undefined
+      const speechTokenIds: number[] = []
+      let sawSpeechTokenIds = false
       try {
         for await (const data of response.iterate()) {
           if (await cancelIfAborted(model, ctx.signal)) continue
@@ -152,6 +155,10 @@ export async function* textToSpeech(params: TtsRequest): AsyncGenerator<TtsOpYie
             sampleRate ??= data.sampleRate
             parts.push(data.outputArray)
           }
+          if (data.speechTokenIds !== undefined) {
+            sawSpeechTokenIds = true
+            speechTokenIds.push(...data.speechTokenIds)
+          }
         }
       } catch (error) {
         rethrowUnlessCancelled(error, ctx.signal)
@@ -160,7 +167,8 @@ export async function* textToSpeech(params: TtsRequest): AsyncGenerator<TtsOpYie
       if (ctx.signal.aborted) return finish(modelStart, response, true)
       yield {
         ...pcmFrame(concatPcm(parts)),
-        ...(sampleRate !== undefined ? { sampleRate } : {})
+        ...(sampleRate !== undefined ? { sampleRate } : {}),
+        ...(sawSpeechTokenIds ? { speechTokenIds } : {})
       }
       return finish(modelStart, response, false)
     }
@@ -170,7 +178,9 @@ export async function* textToSpeech(params: TtsRequest): AsyncGenerator<TtsOpYie
         if (await cancelIfAborted(model, ctx.signal)) continue
         // lunte-disable-next-line eqeqeq -- `== null` intentionally matches null and undefined
         if (data.outputArray == null) continue
-        if (data.outputArray.length === 0) continue
+        // An empty chunk is a native end marker, unless it carries the run's
+        // speech tokens (CosyVoice3 native streaming), which the caller wants.
+        if (data.outputArray.length === 0 && !carriesSpeechTokenIds(data)) continue
         yield { ...pcmFrame(data.outputArray), ...chunkMetadata(data) }
       }
     } catch (error) {
@@ -209,12 +219,20 @@ export async function* textToSpeech(params: TtsRequest): AsyncGenerator<TtsOpYie
     const parts: ArrayLike<number>[] = []
     let sampleRate: number | undefined
     let replyText: string | undefined
+    // CosyVoice3 batch output carries the run's speech tokens with the PCM;
+    // under native chunk streaming they arrive on the final empty chunk.
+    const speechTokenIds: number[] = []
+    let sawSpeechTokenIds = false
 
     try {
       for await (const data of response.iterate()) {
         if (await cancelIfAborted(model, ctx.signal)) continue
         sampleRate ??= data.sampleRate
         if (data.text !== undefined) replyText = data.text
+        if (data.speechTokenIds !== undefined) {
+          sawSpeechTokenIds = true
+          speechTokenIds.push(...data.speechTokenIds)
+        }
         parts.push(data.outputArray)
       }
     } catch (error) {
@@ -224,7 +242,8 @@ export async function* textToSpeech(params: TtsRequest): AsyncGenerator<TtsOpYie
     yield {
       ...pcmFrame(concatPcm(parts)),
       ...(sampleRate !== undefined ? { sampleRate } : {}),
-      ...(replyText !== undefined ? { text: replyText } : {})
+      ...(replyText !== undefined ? { text: replyText } : {}),
+      ...(sawSpeechTokenIds ? { speechTokenIds } : {})
     }
     return finish(modelStart, response, false)
   }