Realtime text to speech

curl -X POST "https://api.breeze.blue/v1/text-to-speech/voc_xeh3w54cqvnp/realtime-sessions" \
  -H "xi-api-key: $BREEZE_API_KEY" \
  -H "content-type: application/json" \
  -d '{
    "model_id": "breeze-tts-2",
    "language_code": "en",
    "instructions": "Keep turns responsive and conversational.",
    "voice_settings": { "guidance_scale": 4.0 },
    "inactivity_timeout_seconds": 30
  }'

GET wss://api.breeze.blue/v1/text-to-speech/{voice_id}/stream-input?client_secret=...

breeze-realtime-v1
breeze-realtime-token.{client_secret}

import { writeFile } from "node:fs/promises";

const apiKey = process.env.BREEZE_API_KEY;
if (!apiKey) throw new Error("Set BREEZE_API_KEY before running this script");

const voiceId = "voc_xeh3w54cqvnp";
const response = await fetch(
  `https://api.breeze.blue/v1/text-to-speech/${voiceId}/realtime-sessions`,
  {
    method: "POST",
    headers: {
      "content-type": "application/json",
      "xi-api-key": apiKey,
    },
    body: JSON.stringify({
      model_id: "breeze-tts-2",
      inactivity_timeout_seconds: 30,
    }),
  },
);

if (!response.ok) {
  throw new Error(`Session creation failed: ${response.status} ${await response.text()}`);
}

const session = await response.json();
const websocketUrl = session.direct_websocket_url ?? session.websocket_url;
const protocols = session.direct_websocket_url
  ? ["breeze-realtime-v1", `breeze-realtime-token.${session.client_secret}`]
  : undefined;
const ws = new WebSocket(websocketUrl, protocols);
ws.binaryType = "arraybuffer";

const pcmChunks = [];
await new Promise((resolve, reject) => {
  const timeout = setTimeout(() => {
    ws.close();
    reject(new Error("Timed out waiting for turn.done"));
  }, 30_000);

  ws.onerror = () => {
    clearTimeout(timeout);
    ws.close();
    reject(new Error("WebSocket connection failed"));
  };
  ws.onopen = () => {
    if (
      session.direct_websocket_url &&
      ws.protocol !== "breeze-realtime-v1"
    ) {
      clearTimeout(timeout);
      ws.close();
      reject(new Error("Direct WebSocket did not negotiate breeze-realtime-v1"));
    }
  };
  ws.onmessage = (event) => {
    if (typeof event.data !== "string") {
      pcmChunks.push(Buffer.from(event.data));
      return;
    }

    const message = JSON.parse(event.data);
    if (message.type === "session.ready") {
      ws.send(JSON.stringify({ type: "turn.start", turn_id: "turn_1" }));
      ws.send(JSON.stringify({ type: "text.append", text: "Hello from Breeze." }));
      ws.send(JSON.stringify({ type: "text.flush" }));
      ws.send(JSON.stringify({ type: "turn.end" }));
    } else if (message.type === "error") {
      clearTimeout(timeout);
      ws.close();
      reject(new Error(`${message.code}: ${message.message}`));
    } else if (message.type === "turn.done") {
      clearTimeout(timeout);
      ws.send(JSON.stringify({ type: "session.close" }));
      resolve();
    }
  };
});

await writeFile("out.pcm", Buffer.concat(pcmChunks));

{"type":"turn.start","turn_id":"turn_1"}
{"type":"text.append","text":"Hello "}
{"type":"text.flush"}
{"type":"turn.end"}

{"type":"session.update","instructions":"Speak faster and with more energy."}

{"type":"session.updated","instructions":"Speak faster and with more energy."}

{"type":"ping"}

import { writeFile } from "node:fs/promises";
import { BreezeBlueClient } from "@breeze.blue/sdk";

const client = new BreezeBlueClient({
  apiKey: process.env.BREEZE_API_KEY!,
});

const connection = await client.textToSpeech.realtime.connect("voc_xeh3w54cqvnp", {
  modelId: "breeze-tts-2",
  instructions: "Keep the turn warm and natural.",
});

const pcmChunks: Uint8Array[] = [];
const consumer = (async () => {
  for await (const message of connection) {
    if (message.type === "audio") {
      // Forward this chunk to your playback or transport layer immediately.
      pcmChunks.push(message.audio);
    } else if (message.type === "error") {
      throw new Error(`Realtime TTS failed: ${message.code}: ${message.message}`);
    } else if (message.type === "turn.cancelled") {
      throw new Error(`Turn cancelled: ${message.turnId}`);
    } else if (message.type === "turn.done") {
      return;
    }
  }
  throw new Error("Realtime session closed before turn.done");
})();

connection.startTurn("turn_1");
for (const delta of ["Hello ", "from Breeze."]) {
  connection.appendText(delta);
  if (/[.!?]\s*$/.test(delta)) connection.flush();
}
connection.endTurn();

await consumer;
await writeFile(
  "out.pcm",
  Buffer.concat(pcmChunks.map((chunk) => Buffer.from(chunk))),
);
connection.close();

import asyncio
import os
from pathlib import Path

from breeze_blue import BreezeBlue

client = BreezeBlue(api_key=os.environ["BREEZE_API_KEY"])


async def consume_audio(connection) -> bytes:
    audio = bytearray()
    async for event in connection:
        if event["type"] == "audio":
            # Forward this chunk to your playback or transport layer immediately.
            audio.extend(event["audio"])
        elif event["type"] == "error":
            raise RuntimeError(event.get("message"))
        elif event["type"] == "turn.cancelled":
            raise RuntimeError(f"Turn cancelled: {event['turn_id']}")
        elif event["type"] == "turn.done":
            return bytes(audio)
    raise RuntimeError("Realtime session closed before turn.done")


async def speak() -> None:
    async with client.text_to_speech.connect_realtime(
        voice_id="voc_xeh3w54cqvnp",
        model_id="breeze-tts-2",
        instructions="Keep the turn warm and natural.",
    ) as connection:
        consumer = asyncio.create_task(consume_audio(connection))
        await connection.start_turn("turn_1")
        for delta in ("Hello ", "from Breeze."):
            await connection.append_text(delta)
            if delta.rstrip().endswith((".", "!", "?")):
                await connection.flush()
        await connection.end_turn()
        Path("out.pcm").write_bytes(await consumer)


asyncio.run(speak())

const connection = await client.textToSpeech.realtime.connectManaged(
  "voc_xeh3w54cqvnp",
  { modelId: "breeze-tts-2" },
);

await connection.startTurnWhenReady("turn_1");
connection.appendText("Hello from a long-running conversation.");
connection.endTurn();

async with client.text_to_speech.connect_managed(
    voice_id="voc_xeh3w54cqvnp",
    model_id="breeze-tts-2",
) as connection:
    await connection.start_turn("turn_1")
    await connection.append_text("Hello from a long-running conversation.")
    await connection.end_turn()

import { BreezeBlueRealtimeError } from "@breeze.blue/sdk";

const browserClient = new BreezeBlueClient();
const connection = await browserClient.textToSpeech.realtime.connectManaged(
  "voc_xeh3w54cqvnp",
  {
    sessionFactory: async ({ signal }) => {
      const response = await fetch("/api/breeze-realtime-session", {
        method: "POST",
        signal,
      });
      if (!response.ok) {
        const reconnect =
          [408, 425, 429].includes(response.status) || response.status >= 500;
        throw new BreezeBlueRealtimeError("Could not create realtime session", {
          code: "SESSION_FACTORY_ERROR",
          reconnect,
        });
      }
      const session = await response.json();
      return {
        clientSecret: session.client_secret,
        websocketUrl: session.websocket_url,
        directWebsocketUrl: session.direct_websocket_url,
      };
    },
  },
);

// Your backend calls POST /v1/text-to-speech/{voice_id}/realtime-sessions
// and returns the session credentials to the page.
const session = await fetch("/api/realtime-session", {
  method: "POST",
}).then((response) => response.json());

const websocketUrl = session.direct_websocket_url ?? session.websocket_url;
const protocols = session.direct_websocket_url
  ? ["breeze-realtime-v1", `breeze-realtime-token.${session.client_secret}`]
  : undefined;

const audioContext = new AudioContext({ sampleRate: 24000 });
let playhead = audioContext.currentTime;

const ws = new WebSocket(websocketUrl, protocols);
ws.binaryType = "arraybuffer";

ws.onopen = () => {
  if (
    session.direct_websocket_url &&
    ws.protocol !== "breeze-realtime-v1"
  ) {
    ws.close();
    throw new Error("Direct WebSocket did not negotiate breeze-realtime-v1");
  }
};

ws.onmessage = (event) => {
  if (typeof event.data === "string") {
    const message = JSON.parse(event.data);
    if (message.type === "session.ready") {
      ws.send(JSON.stringify({ type: "turn.start", turn_id: "turn_1" }));
      ws.send(JSON.stringify({ type: "text.append", text: "Hello from the browser." }));
      ws.send(JSON.stringify({ type: "text.flush" }));
      ws.send(JSON.stringify({ type: "turn.end" }));
    } else if (message.type === "turn.done") {
      ws.send(JSON.stringify({ type: "session.close" }));
    }
    return;
  }
  // Binary frame: pcm_s16le / 24000 Hz / mono.
  const pcm = new Int16Array(event.data);
  const samples = Float32Array.from(pcm, (sample) => sample / 32768);
  const buffer = audioContext.createBuffer(1, samples.length, 24000);
  buffer.copyToChannel(samples, 0);
  const source = audioContext.createBufferSource();
  source.buffer = buffer;
  source.connect(audioContext.destination);
  playhead = Math.max(playhead, audioContext.currentTime);
  source.start(playhead);
  playhead += buffer.duration;
};

breeze version
breeze tts realtime "Hello from Breeze" --voice voc_xeh3w54cqvnp --model breeze-tts-2 --no-play --file out.wav --agent
printf 'Hello\nHow are you?\n' | breeze tts realtime --stdin --voice voc_xeh3w54cqvnp --model breeze-tts-2
printf '%s\n' \
  '{"type":"turn","text":"First turn."}' \
  '{"type":"session.update","instructions":"Speak faster and with more energy."}' \
  '{"type":"turn","text":"Second turn."}' |
  breeze tts realtime --stdin --stdin-format jsonl --voice voc_xeh3w54cqvnp --model breeze-tts-2 --agent
cat events.jsonl | breeze tts realtime --events --voice voc_xeh3w54cqvnp --model breeze-tts-2 --output json --no-play