{
  "version": 1,
  "language": "en",
  "title": "One chat request across engine processes",
  "description": "Two source-derived traces follow one streaming chat request on one GPU. vLLM detokenizes in the API-server process. SGLang sends token IDs to a separate detokenizer process. Select either trace on this page.",
  "caption": "Author-created trace from vLLM v0.32.0 and SGLang v0.5.23 with default settings and one GPU. With tensor parallelism above 1, both engines add worker or scheduler processes per GPU. Playback time is not measured latency.",
  "alt": "Flow diagram: a chat request goes from the frontend to the scheduler and GPU worker, then returns through the frontend or a separate detokenizer",
  "stateLabels": {
    "frontend": "Frontend process",
    "engine": "Engine process",
    "transport": "Last hop",
    "stop": "Stop-string check",
    "processes": "Processes at one GPU"
  },
  "targets": {
    "frontend": {
      "id": "hops-frontend",
      "kind": "node",
      "label": "Frontend"
    },
    "scheduler": {
      "id": "hops-scheduler",
      "kind": "node",
      "label": "Scheduler"
    },
    "worker": {
      "id": "hops-worker",
      "kind": "node",
      "label": "GPU worker"
    },
    "detok": {
      "id": "hops-detok",
      "kind": "node",
      "label": "Detokenizer"
    },
    "request": {
      "id": "hops-request",
      "kind": "edge",
      "label": "Send token IDs"
    },
    "directOut": {
      "id": "hops-direct-out",
      "kind": "edge",
      "label": "Return new tokens"
    },
    "launch": {
      "id": "hops-launch",
      "kind": "edge",
      "label": "Launch batch"
    },
    "result": {
      "id": "hops-result",
      "kind": "edge",
      "label": "Return sampled tokens"
    },
    "toDetok": {
      "id": "hops-to-detok",
      "kind": "edge",
      "label": "Send token IDs to detokenizer"
    },
    "textOut": {
      "id": "hops-text-out",
      "kind": "edge",
      "label": "Return text"
    }
  },
  "sources": [
    {
      "title": "Original author diagram",
      "file": "diagram.svg",
      "sha256": "fb45e23501155242baf987c28329e5bc572df518fb784ace09f30d5f4fbfc8fa"
    },
    {
      "title": "vLLM v0.32.0 frontend client sockets",
      "url": "https://github.com/vllm-project/vllm/blob/v0.32.0/vllm/v1/engine/core_client.py#L659-L669"
    },
    {
      "title": "vLLM v0.32.0 output processor",
      "url": "https://github.com/vllm-project/vllm/blob/v0.32.0/vllm/v1/engine/output_processor.py#L765-L775"
    },
    {
      "title": "vLLM v0.32.0 single-process executor",
      "url": "https://github.com/vllm-project/vllm/blob/v0.32.0/vllm/v1/executor/uniproc_executor.py#L51-L69"
    },
    {
      "title": "SGLang v0.5.23 process layout",
      "url": "https://github.com/sgl-project/sglang/blob/v0.5.23/python/sglang/srt/entrypoints/engine.py#L248-L260"
    },
    {
      "title": "SGLang v0.5.23 stop-string check",
      "url": "https://github.com/sgl-project/sglang/blob/v0.5.23/python/sglang/srt/managers/schedule_batch.py#L1910-L1925"
    },
    {
      "title": "Article evidence and version ledger",
      "file": "../../illustrations/SOURCE.md"
    }
  ],
  "limitations": [
    "The traces assume default serve settings, one node, one GPU and a streaming chat request.",
    "With tensor parallelism above 1, vLLM adds one worker process per GPU and SGLang adds one scheduler process per TP rank.",
    "Steps are a teaching order. Playback duration does not measure latency."
  ],
  "scenarios": [
    {
      "id": "vllm",
      "title": "vLLM: frontend and EngineCore",
      "description": "The API server tokenizes and detokenizes. EngineCore schedules and runs the GPU worker in the same process.",
      "initial": {
        "frontend": "idle",
        "engine": "idle",
        "transport": "none",
        "stop": "frontend OutputProcessor",
        "processes": "2: API server, EngineCore"
      },
      "unselected": [
        "detok",
        "toDetok",
        "textOut"
      ],
      "steps": [
        {
          "title": "Template and tokenize",
          "description": "OnlineRenderer applies the chat template and tokenizes in the API-server process.",
          "duration": 3000,
          "active": [
            "frontend"
          ],
          "paths": [],
          "state": {
            "frontend": "chat template and tokenize",
            "engine": "idle",
            "transport": "none",
            "stop": "frontend OutputProcessor",
            "processes": "2: API server, EngineCore"
          }
        },
        {
          "title": "Send the request over ZMQ",
          "description": "AsyncLLM sends an EngineCoreRequest through a ZMQ ROUTER socket. EngineCore adds it to the waiting queue.",
          "duration": 3000,
          "active": [
            "scheduler"
          ],
          "paths": [
            "request"
          ],
          "state": {
            "frontend": "waits for outputs",
            "engine": "request in waiting queue",
            "transport": "ZMQ ROUTER to DEALER, msgpack",
            "stop": "frontend OutputProcessor",
            "processes": "2: API server, EngineCore"
          }
        },
        {
          "title": "Launch the batch",
          "description": "step_with_batch_queue schedules the step. The uni executor calls the GPU worker inside EngineCore.",
          "duration": 3000,
          "active": [
            "worker"
          ],
          "paths": [
            "launch"
          ],
          "state": {
            "frontend": "waits for outputs",
            "engine": "model runner executes",
            "transport": "function call",
            "stop": "frontend OutputProcessor",
            "processes": "2: API server, EngineCore"
          }
        },
        {
          "title": "Update requests",
          "description": "update_from_output records sampled tokens and checks EOS, stop token IDs and max_tokens.",
          "duration": 3000,
          "active": [
            "scheduler"
          ],
          "paths": [
            "result"
          ],
          "state": {
            "frontend": "waits for outputs",
            "engine": "check_stop: EOS and length",
            "transport": "function call",
            "stop": "frontend OutputProcessor",
            "processes": "2: API server, EngineCore"
          }
        },
        {
          "title": "Detokenize in the frontend",
          "description": "EngineCore pushes new tokens over ZMQ. The frontend detokenizes, matches stop strings and streams text. A stop-string hit sends an abort back.",
          "duration": 3600,
          "active": [
            "frontend"
          ],
          "paths": [
            "directOut"
          ],
          "state": {
            "frontend": "detokenize and stream text",
            "engine": "next step",
            "transport": "ZMQ PUSH to PULL, msgpack",
            "stop": "frontend OutputProcessor",
            "processes": "2: API server, EngineCore"
          }
        }
      ]
    },
    {
      "id": "sglang",
      "title": "SGLang: tokenizer, scheduler, detokenizer",
      "description": "The main process tokenizes. The Scheduler process holds the GPU worker and checks stop strings. A separate process detokenizes.",
      "initial": {
        "frontend": "idle",
        "engine": "idle",
        "transport": "none",
        "stop": "Scheduler",
        "processes": "3: main, Scheduler, Detokenizer"
      },
      "unselected": [
        "directOut"
      ],
      "steps": [
        {
          "title": "Template and tokenize",
          "description": "The chat layer and TokenizerManager tokenize in the main process.",
          "duration": 3000,
          "active": [
            "frontend"
          ],
          "paths": [],
          "state": {
            "frontend": "chat template and tokenize",
            "engine": "idle",
            "transport": "none",
            "stop": "Scheduler",
            "processes": "3: main, Scheduler, Detokenizer"
          }
        },
        {
          "title": "Push to the Scheduler",
          "description": "TokenizerManager pushes the tokenized request over ZMQ. TP rank 0 receives it.",
          "duration": 3000,
          "active": [
            "scheduler"
          ],
          "paths": [
            "request"
          ],
          "state": {
            "frontend": "waits for text",
            "engine": "request in waiting queue",
            "transport": "ZMQ PUSH to PULL",
            "stop": "Scheduler",
            "processes": "3: main, Scheduler, Detokenizer"
          }
        },
        {
          "title": "Launch the batch",
          "description": "event_loop_overlap picks the batch and runs TpModelWorker inside the Scheduler process.",
          "duration": 3000,
          "active": [
            "worker"
          ],
          "paths": [
            "launch"
          ],
          "state": {
            "frontend": "waits for text",
            "engine": "TpModelWorker executes",
            "transport": "function call",
            "stop": "Scheduler",
            "processes": "3: main, Scheduler, Detokenizer"
          }
        },
        {
          "title": "Process results and stop strings",
          "description": "process_batch_result updates each request. The Scheduler checks stop strings on a decoded tail.",
          "duration": 3000,
          "active": [
            "scheduler"
          ],
          "paths": [
            "result"
          ],
          "state": {
            "frontend": "waits for text",
            "engine": "finish checks incl. stop strings",
            "transport": "function call",
            "stop": "Scheduler",
            "processes": "3: main, Scheduler, Detokenizer"
          }
        },
        {
          "title": "Send token IDs to the detokenizer",
          "description": "The Scheduler pushes a BatchTokenIDOutput over ZMQ. DetokenizerManager decodes incrementally.",
          "duration": 3000,
          "active": [
            "detok"
          ],
          "paths": [
            "toDetok"
          ],
          "state": {
            "frontend": "waits for text",
            "engine": "next step",
            "transport": "ZMQ PUSH to PULL",
            "stop": "Scheduler",
            "processes": "3: main, Scheduler, Detokenizer"
          }
        },
        {
          "title": "Return text to the frontend",
          "description": "The detokenizer pushes text to TokenizerManager, which streams it to the client.",
          "duration": 3000,
          "active": [
            "frontend"
          ],
          "paths": [
            "textOut"
          ],
          "state": {
            "frontend": "stream text",
            "engine": "next step",
            "transport": "ZMQ PUSH to PULL",
            "stop": "Scheduler",
            "processes": "3: main, Scheduler, Detokenizer"
          }
        }
      ]
    }
  ]
}