{
  "version": 1,
  "language": "en",
  "title": "One speech request through three model stages",
  "description": "Two source-derived traces follow one streaming Qwen3-Omni request that returns text and speech. vLLM-Omni runs each stage as a vLLM engine. SGLang-Omni runs each stage as a node in a coordinator graph. Select either trace on this page.",
  "caption": "Author-created trace from vLLM-Omni v0.32.0 and SGLang-Omni v0.2.0 default Qwen3-Omni configurations. Encoder and preprocessing stages are not drawn. Playback time is not measured latency.",
  "alt": "Flow diagram: a Qwen3-Omni request streams from Thinker to Talker to Code2Wav, with text returned directly or through a separate decode stage",
  "stateLabels": {
    "controller": "Controller",
    "thinker": "Thinker",
    "talker": "Talker",
    "vocoder": "Code2Wav",
    "handoff": "Thinker-to-Talker payload",
    "transport": "Stage-to-stage transport",
    "client": "Client stream"
  },
  "targets": {
    "frontend": {
      "id": "omni-frontend",
      "kind": "node",
      "label": "Frontend"
    },
    "thinker": {
      "id": "omni-thinker",
      "kind": "node",
      "label": "Thinker"
    },
    "talker": {
      "id": "omni-talker",
      "kind": "node",
      "label": "Talker"
    },
    "vocoder": {
      "id": "omni-vocoder",
      "kind": "node",
      "label": "Code2Wav"
    },
    "decode": {
      "id": "omni-decode",
      "kind": "node",
      "label": "Text decode"
    },
    "submit": {
      "id": "omni-submit",
      "kind": "edge",
      "label": "Submit request"
    },
    "thinkerText": {
      "id": "omni-thinker-text",
      "kind": "edge",
      "label": "Return text tokens"
    },
    "decodeText": {
      "id": "omni-decode-text",
      "kind": "edge",
      "label": "Return text delta"
    },
    "thinkerDecode": {
      "id": "omni-thinker-decode",
      "kind": "edge",
      "label": "Stream token IDs to decode"
    },
    "thinkerTalker": {
      "id": "omni-thinker-talker",
      "kind": "edge",
      "label": "Stream Thinker chunk"
    },
    "talkerVocoder": {
      "id": "omni-talker-vocoder",
      "kind": "edge",
      "label": "Stream codec chunk"
    },
    "audio": {
      "id": "omni-audio",
      "kind": "edge",
      "label": "Return audio chunk"
    },
    "presubmit": {
      "id": "omni-presubmit",
      "kind": "edge",
      "label": "Pre-submit placeholder request"
    }
  },
  "sources": [
    {
      "title": "Original author diagram",
      "file": "diagram.svg",
      "sha256": "984d8b934cc744fa72cb1fdad6be9e4b9dd36097417edf1915cb9cae06fdc3dc"
    },
    {
      "title": "vLLM-Omni v0.32.0 Qwen3-Omni pipeline",
      "url": "https://github.com/vllm-project/vllm-omni/blob/v0.32.0/vllm_omni/model_executor/models/qwen3_omni/pipeline.py"
    },
    {
      "title": "vLLM-Omni v0.32.0 Qwen3-Omni deploy configuration",
      "url": "https://github.com/vllm-project/vllm-omni/blob/v0.32.0/vllm_omni/deploy/qwen3_omni_moe.yaml"
    },
    {
      "title": "vLLM-Omni v0.32.0 Orchestrator",
      "url": "https://github.com/vllm-project/vllm-omni/blob/v0.32.0/vllm_omni/engine/orchestrator.py"
    },
    {
      "title": "SGLang-Omni v0.2.0 Qwen3-Omni stage graph",
      "url": "https://github.com/sgl-project/sglang-omni/blob/v0.2.0/sglang_omni/models/qwen3_omni/config.py"
    },
    {
      "title": "SGLang-Omni v0.2.0 request builders",
      "url": "https://github.com/sgl-project/sglang-omni/blob/v0.2.0/sglang_omni/models/qwen3_omni/request_builders.py"
    },
    {
      "title": "Article evidence and version ledger",
      "file": "../../illustrations/SOURCE.md"
    }
  ],
  "limitations": [
    "The trace uses each project's default Qwen3-Omni configuration at the pinned tag. Other deploy files can change GPU placement, transport and chunk sizes.",
    "Preprocessing and encoder stages are omitted. In SGLang-Omni they are separate stages. In vLLM-Omni they run inside the Thinker engine.",
    "Steps are a teaching order. Text and speech chunks overlap in time on a real server. Playback duration does not measure latency."
  ],
  "scenarios": [
    {
      "id": "vllm-omni",
      "title": "vLLM-Omni: each stage is a vLLM engine",
      "description": "An Orchestrator thread in the API process submits work to three engine-core processes. Stage schedulers write chunks into a shared-memory connector themselves.",
      "initial": {
        "controller": "Orchestrator thread",
        "thinker": "idle",
        "talker": "idle",
        "vocoder": "idle",
        "handoff": "none",
        "transport": "none",
        "client": "no output"
      },
      "unselected": [
        "decode",
        "thinkerDecode",
        "decodeText"
      ],
      "steps": [
        {
          "title": "Submit to the Thinker engine",
          "description": "The Orchestrator sends an engine-core request over ZMQ to the Thinker process on GPU 0.",
          "duration": 3000,
          "active": [
            "thinker"
          ],
          "paths": [
            "submit"
          ],
          "state": {
            "controller": "Orchestrator thread",
            "thinker": "prefill on GPU 0",
            "talker": "idle",
            "vocoder": "idle",
            "handoff": "none",
            "transport": "ZMQ request",
            "client": "no output"
          }
        },
        {
          "title": "Pre-submit placeholder requests",
          "description": "The Orchestrator also submits placeholder requests to the downstream stages. They wait in the WAITING_FOR_CHUNK status.",
          "duration": 3000,
          "active": [
            "talker"
          ],
          "paths": [
            "presubmit"
          ],
          "state": {
            "controller": "Orchestrator thread",
            "thinker": "prefill on GPU 0",
            "talker": "WAITING_FOR_CHUNK",
            "vocoder": "WAITING_FOR_CHUNK",
            "handoff": "none",
            "transport": "ZMQ request",
            "client": "no output"
          }
        },
        {
          "title": "Stream text and the first chunk",
          "description": "Each Thinker step returns text tokens to the frontend. The Thinker scheduler writes chunk 0 to shared memory: prompt embeddings, hidden states and token IDs.",
          "duration": 3600,
          "active": [
            "frontend",
            "talker"
          ],
          "paths": [
            "thinkerText",
            "thinkerTalker"
          ],
          "state": {
            "controller": "Orchestrator thread",
            "thinker": "decoding text",
            "talker": "decoding codec frames on GPU 1",
            "vocoder": "WAITING_FOR_CHUNK",
            "handoff": "embeddings and hidden states",
            "transport": "SharedMemoryConnector",
            "client": "text deltas"
          }
        },
        {
          "title": "Send codec chunks to Code2Wav",
          "description": "The Talker groups codec frames: 4 frames first, then 25 frames, with left context. Later Thinker chunks carry one token embedding each.",
          "duration": 3300,
          "active": [
            "vocoder"
          ],
          "paths": [
            "talkerVocoder"
          ],
          "state": {
            "controller": "Orchestrator thread",
            "thinker": "decoding text",
            "talker": "decoding codec frames on GPU 1",
            "vocoder": "non-AR decode on GPU 1",
            "handoff": "one token embedding per step",
            "transport": "SharedMemoryConnector",
            "client": "text deltas"
          }
        },
        {
          "title": "Return audio to the client",
          "description": "Code2Wav output returns over ZMQ to the Orchestrator. The API sends it as a stream chunk tagged audio.",
          "duration": 3000,
          "active": [
            "frontend"
          ],
          "paths": [
            "audio"
          ],
          "state": {
            "controller": "Orchestrator thread",
            "thinker": "decoding text",
            "talker": "decoding codec frames on GPU 1",
            "vocoder": "non-AR decode on GPU 1",
            "handoff": "one token embedding per step",
            "transport": "SharedMemoryConnector",
            "client": "text and audio chunks"
          }
        }
      ]
    },
    {
      "id": "sglang-omni",
      "title": "SGLang-Omni: stages are nodes in a graph",
      "description": "A Coordinator in the runner process sends messages over ZMQ. A router selects the data path for each pair of stages.",
      "initial": {
        "controller": "Coordinator",
        "thinker": "idle",
        "talker": "idle",
        "vocoder": "idle",
        "handoff": "none",
        "transport": "none",
        "client": "no output"
      },
      "unselected": [
        "presubmit",
        "thinkerText"
      ],
      "steps": [
        {
          "title": "Submit through the stage graph",
          "description": "The Coordinator sends the request over ZMQ. Preprocessing and encoder stages run first. They are not drawn. The Thinker receives the embeddings.",
          "duration": 3300,
          "active": [
            "thinker"
          ],
          "paths": [
            "submit"
          ],
          "state": {
            "controller": "Coordinator",
            "thinker": "prefill on GPU 0",
            "talker": "idle",
            "vocoder": "idle",
            "handoff": "none",
            "transport": "ZMQ messages",
            "client": "no output"
          }
        },
        {
          "title": "Fan out token IDs",
          "description": "Each Thinker step streams a token-ID tensor to Text decode and to the Talker. The first Talker chunk also carries prompt hidden rows.",
          "duration": 3600,
          "active": [
            "decode",
            "talker"
          ],
          "paths": [
            "thinkerDecode",
            "thinkerTalker"
          ],
          "state": {
            "controller": "Coordinator",
            "thinker": "decoding text",
            "talker": "re-embeds IDs on GPU 1",
            "vocoder": "idle",
            "handoff": "token IDs",
            "transport": "CUDA IPC relay",
            "client": "no output"
          }
        },
        {
          "title": "Stream text deltas",
          "description": "Text decode is a CPU stage. It sends text deltas to the Coordinator, which streams them to the client.",
          "duration": 3000,
          "active": [
            "frontend"
          ],
          "paths": [
            "decodeText"
          ],
          "state": {
            "controller": "Coordinator",
            "thinker": "decoding text",
            "talker": "decoding codec frames on GPU 1",
            "vocoder": "idle",
            "handoff": "token IDs",
            "transport": "CUDA IPC relay",
            "client": "text deltas"
          }
        },
        {
          "title": "Send codec chunks to Code2Wav",
          "description": "The Talker sends 10-frame codec chunks through the pooled CUDA IPC relay, from GPU 1 to GPU 0.",
          "duration": 3300,
          "active": [
            "vocoder"
          ],
          "paths": [
            "talkerVocoder"
          ],
          "state": {
            "controller": "Coordinator",
            "thinker": "decoding text",
            "talker": "decoding codec frames on GPU 1",
            "vocoder": "streaming vocoder on GPU 0",
            "handoff": "token IDs",
            "transport": "pooled CUDA IPC relay",
            "client": "text deltas"
          }
        },
        {
          "title": "Merge audio into the stream",
          "description": "Code2Wav streams audio chunks to the Coordinator. The request ends when both terminal stages, Text decode and Code2Wav, finish.",
          "duration": 3300,
          "active": [
            "frontend"
          ],
          "paths": [
            "audio"
          ],
          "state": {
            "controller": "Coordinator",
            "thinker": "decoding text",
            "talker": "decoding codec frames on GPU 1",
            "vocoder": "streaming vocoder on GPU 0",
            "handoff": "token IDs",
            "transport": "pooled CUDA IPC relay",
            "client": "text and audio chunks"
          }
        }
      ]
    }
  ]
}