{
  "version": 1,
  "language": "en",
  "title": "Serial and overlapped scheduler loops",
  "description": "Two source-derived traces follow one SGLang scheduler iteration. The serial loop processes batch k right after it runs. The overlap loop queues batch k and processes batch k−1 while the GPU runs batch k.",
  "caption": "Author-created trace of SGLang v0.5.23 event_loop_normal and event_loop_overlap. vLLM, TensorRT-LLM and LMDeploy use the same one-step-ahead idea with different code. Playback time is not measured latency.",
  "alt": "Flow diagram: an SGLang scheduler picks and launches batch k, the GPU reads inputs from a FutureMap, and the CPU processes the previous batch's results",
  "stateLabels": {
    "cpu": "CPU work",
    "gpu": "GPU work",
    "inputs": "Next input IDs",
    "queue": "Result queue",
    "idle": "GPU idle during CPU work"
  },
  "targets": {
    "pick": {
      "id": "overlap-pick",
      "kind": "node",
      "label": "Pick batch"
    },
    "launch": {
      "id": "overlap-launch",
      "kind": "node",
      "label": "Launch step"
    },
    "queue": {
      "id": "overlap-queue",
      "kind": "node",
      "label": "Result queue"
    },
    "process": {
      "id": "overlap-process",
      "kind": "node",
      "label": "Process results"
    },
    "gpu": {
      "id": "overlap-gpu",
      "kind": "node",
      "label": "GPU forward"
    },
    "future": {
      "id": "overlap-future",
      "kind": "node",
      "label": "FutureMap"
    },
    "pickLaunch": {
      "id": "overlap-pick-launch",
      "kind": "edge",
      "label": "Hand batch to launch"
    },
    "launchGpu": {
      "id": "overlap-launch-gpu",
      "kind": "edge",
      "label": "Launch the forward"
    },
    "launchQueue": {
      "id": "overlap-launch-queue",
      "kind": "edge",
      "label": "Queue the result handle"
    },
    "queueProcess": {
      "id": "overlap-queue-process",
      "kind": "edge",
      "label": "Pop the previous result"
    },
    "gpuFuture": {
      "id": "overlap-gpu-future",
      "kind": "edge",
      "label": "Write sampled tokens"
    },
    "futureGpu": {
      "id": "overlap-future-gpu",
      "kind": "edge",
      "label": "Read next input IDs"
    },
    "copy": {
      "id": "overlap-copy",
      "kind": "edge",
      "label": "Copy results to CPU"
    },
    "loop": {
      "id": "overlap-loop",
      "kind": "edge",
      "label": "Start the next iteration"
    }
  },
  "sources": [
    {
      "title": "Original author diagram",
      "file": "diagram.svg",
      "sha256": "841fa1c5fa9689e91205f0731b67090a68c042bd106138fab8d69c6bb1cf5bd8"
    },
    {
      "title": "SGLang v0.5.23 event_loop_normal",
      "url": "https://github.com/sgl-project/sglang/blob/v0.5.23/python/sglang/srt/managers/scheduler.py#L2026-L2058"
    },
    {
      "title": "SGLang v0.5.23 event_loop_overlap",
      "url": "https://github.com/sgl-project/sglang/blob/v0.5.23/python/sglang/srt/managers/scheduler.py#L2061-L2134"
    },
    {
      "title": "SGLang v0.5.23 FutureMap input gather",
      "url": "https://github.com/sgl-project/sglang/blob/v0.5.23/python/sglang/srt/managers/overlap_utils.py#L113-L148"
    },
    {
      "title": "SGLang v0.5.23 result copy stream",
      "url": "https://github.com/sgl-project/sglang/blob/v0.5.23/python/sglang/srt/managers/scheduler.py#L5018-L5042"
    },
    {
      "title": "Article evidence and version ledger",
      "file": "../../illustrations/SOURCE.md"
    }
  ],
  "limitations": [
    "The trace shows a plain decode batch. Speculative decoding, grammars and pipeline parallelism add other branches.",
    "The FutureMap relay is used in both loops at this tag. The loops differ in when results are processed.",
    "Steps are a teaching order. Playback duration does not measure latency or GPU idle time."
  ],
  "scenarios": [
    {
      "id": "serial",
      "title": "event_loop_normal: process each batch at once",
      "description": "The scheduler runs batch k, then processes its results before it picks the next batch. The GPU waits during that CPU work.",
      "initial": {
        "cpu": "idle",
        "gpu": "idle",
        "inputs": "from FutureMap",
        "queue": "not used",
        "idle": "yes"
      },
      "unselected": [
        "queue",
        "launchQueue",
        "queueProcess"
      ],
      "steps": [
        {
          "title": "Pick batch k",
          "description": "get_next_batch_to_run selects batch k and allocates KV slots.",
          "duration": 2800,
          "active": [
            "pick"
          ],
          "paths": [],
          "state": {
            "cpu": "schedule batch k",
            "gpu": "idle",
            "inputs": "from FutureMap",
            "queue": "not used",
            "idle": "yes"
          }
        },
        {
          "title": "Launch step k",
          "description": "run_batch prepares the forward for batch k.",
          "duration": 2600,
          "active": [
            "launch"
          ],
          "paths": [
            "pickLaunch"
          ],
          "state": {
            "cpu": "run_batch(k)",
            "gpu": "idle",
            "inputs": "from FutureMap",
            "queue": "not used",
            "idle": "yes"
          }
        },
        {
          "title": "Run the forward",
          "description": "The forward gathers its input IDs from the FutureMap on the GPU, then runs batch k.",
          "duration": 3000,
          "active": [
            "gpu"
          ],
          "paths": [
            "launchGpu",
            "futureGpu"
          ],
          "state": {
            "cpu": "waits for batch k",
            "gpu": "forward k",
            "inputs": "batch k inputs gathered",
            "queue": "not used",
            "idle": "no"
          }
        },
        {
          "title": "Stash the sampled tokens",
          "description": "The sampled tokens of batch k are written into the FutureMap for the next step.",
          "duration": 2600,
          "active": [
            "future"
          ],
          "paths": [
            "gpuFuture"
          ],
          "state": {
            "cpu": "waits for batch k",
            "gpu": "forward k",
            "inputs": "batch k tokens stored",
            "queue": "not used",
            "idle": "no"
          }
        },
        {
          "title": "Process batch k",
          "description": "process_batch_result waits for the copy of batch k, then updates requests, checks stops and streams output.",
          "duration": 3200,
          "active": [
            "process"
          ],
          "paths": [
            "copy"
          ],
          "state": {
            "cpu": "process_batch_result(k)",
            "gpu": "idle",
            "inputs": "batch k tokens stored",
            "queue": "not used",
            "idle": "yes"
          }
        },
        {
          "title": "Start the next iteration",
          "description": "Only now does the loop pick batch k+1. The GPU stays idle until the next launch.",
          "duration": 2800,
          "active": [
            "pick"
          ],
          "paths": [
            "loop"
          ],
          "state": {
            "cpu": "schedule batch k+1",
            "gpu": "idle",
            "inputs": "batch k tokens stored",
            "queue": "not used",
            "idle": "yes"
          }
        }
      ]
    },
    {
      "id": "overlap",
      "title": "event_loop_overlap: process the previous batch",
      "description": "The scheduler launches batch k, queues its result handle and processes batch k−1 while the GPU runs batch k.",
      "initial": {
        "cpu": "idle",
        "gpu": "forward k−1",
        "inputs": "batch k−1 tokens pending",
        "queue": "k−1",
        "idle": "no"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Pick batch k",
          "description": "get_next_batch_to_run selects batch k. Some of its input tokens are not yet known on the CPU.",
          "duration": 2800,
          "active": [
            "pick"
          ],
          "paths": [],
          "state": {
            "cpu": "schedule batch k",
            "gpu": "forward k−1",
            "inputs": "batch k−1 tokens pending",
            "queue": "k−1",
            "idle": "no"
          }
        },
        {
          "title": "Launch step k",
          "description": "run_batch puts the forward for batch k on the forward stream.",
          "duration": 2600,
          "active": [
            "launch"
          ],
          "paths": [
            "pickLaunch"
          ],
          "state": {
            "cpu": "run_batch(k)",
            "gpu": "forward k−1",
            "inputs": "batch k−1 tokens pending",
            "queue": "k−1",
            "idle": "no"
          }
        },
        {
          "title": "Gather inputs on the GPU",
          "description": "The forward for batch k reads the tokens that batch k−1 sampled, directly from the FutureMap.",
          "duration": 3000,
          "active": [
            "gpu"
          ],
          "paths": [
            "launchGpu",
            "futureGpu"
          ],
          "state": {
            "cpu": "run_batch(k) returns",
            "gpu": "forward k",
            "inputs": "batch k−1 tokens gathered",
            "queue": "k−1",
            "idle": "no"
          }
        },
        {
          "title": "Queue the handle, stash tokens",
          "description": "The CPU appends batch k to result_queue. On the GPU, batch k's sampled tokens go into the FutureMap.",
          "duration": 3000,
          "active": [
            "queue",
            "future"
          ],
          "paths": [
            "launchQueue",
            "gpuFuture"
          ],
          "state": {
            "cpu": "result_queue.append(k)",
            "gpu": "forward k",
            "inputs": "batch k tokens stored",
            "queue": "k−1, k",
            "idle": "no"
          }
        },
        {
          "title": "Process batch k−1 during forward k",
          "description": "pop_and_process takes batch k−1, waits for its finished copy, and updates requests while the GPU runs batch k.",
          "duration": 3400,
          "active": [
            "process"
          ],
          "paths": [
            "queueProcess",
            "copy"
          ],
          "state": {
            "cpu": "process_batch_result(k−1)",
            "gpu": "forward k",
            "inputs": "batch k tokens stored",
            "queue": "k",
            "idle": "no"
          }
        },
        {
          "title": "Start the next iteration",
          "description": "The loop picks batch k+1 while forward k may still run. A request that finished in batch k−1 can be in batch k. Its extra token is skipped later.",
          "duration": 3400,
          "active": [
            "pick"
          ],
          "paths": [
            "loop"
          ],
          "state": {
            "cpu": "schedule batch k+1",
            "gpu": "forward k",
            "inputs": "batch k tokens pending",
            "queue": "k",
            "idle": "no"
          }
        }
      ]
    }
  ]
}