{
  "version": 1,
  "language": "en",
  "title": "Serving iteration: rescheduling or release",
  "description": "Tokenized state enters scheduling, KV allocation/reuse and a selected model batch. Sampling gives new IDs for streaming. Then the engine updates request state. The scheduler can schedule an unfinished request again. When a request is completed, it releases its references. Shared or reusable physical blocks may stay.",
  "caption": "The engine does scheduling, KV management, execution, sampling and state updates again until the request is completed. The constructed example continues one time, then stops. Completion releases request references. It does not necessarily free shared or reusable physical blocks. The author made this explanatory diagram.",
  "alt": "Serving-loop flow: tokenized requests use the scheduler, KV allocation, model executor and sampling, then continue or release cache references.",
  "sources": [
    {
      "path": "content/blog/2026-06-11_life-of-a-request-vllm/post.md",
      "sha256": "158aa77041646bca8aba059344c4b6b0a6b451329b4a423e9c8027db8ba0304a",
      "description": "Current article explanation, equations, assumptions and retained citations."
    },
    {
      "path": "research/dynamic-mechanisms-2026-10-21/before/content/blog/2026-06-11_life-of-a-request-vllm/illustrations/06-serving-loop-algorithm.svg",
      "sha256": "0a4a2894ebc7474141a4f50fe77120c74342740be2c51e44b1164f4e8018a947",
      "description": "Frozen prior illustration; exact pre-migration bytes retained."
    },
    {
      "path": "content/blog/2026-06-11_life-of-a-request-vllm/illustrations/source-notes.json",
      "description": "Existing article illustration provenance, retained unchanged."
    }
  ],
  "limitations": [
    "The execution is an example. Playback duration is not measured runtime."
  ],
  "targets": {
    "state": {
      "id": "mechanism-state",
      "kind": "node",
      "label": "Tokenized state"
    },
    "schedule": {
      "id": "mechanism-schedule",
      "kind": "node",
      "label": "Scheduler"
    },
    "kv": {
      "id": "mechanism-kv",
      "kind": "node",
      "label": "KV allocation / reuse"
    },
    "execute": {
      "id": "mechanism-execute",
      "kind": "node",
      "label": "Model executor"
    },
    "sample": {
      "id": "mechanism-sample",
      "kind": "node",
      "label": "Sample / stream"
    },
    "update": {
      "id": "mechanism-update",
      "kind": "node",
      "label": "Update state"
    },
    "release": {
      "id": "mechanism-release",
      "kind": "node",
      "label": "Release references"
    },
    "admit": {
      "id": "mechanism-admit",
      "kind": "edge",
      "label": "admit"
    },
    "allocate": {
      "id": "mechanism-allocate",
      "kind": "edge",
      "label": "allocate"
    },
    "run": {
      "id": "mechanism-run",
      "kind": "edge",
      "label": "run"
    },
    "sample-output": {
      "id": "mechanism-sample-output",
      "kind": "edge",
      "label": "sample-output"
    },
    "update-output": {
      "id": "mechanism-update-output",
      "kind": "edge",
      "label": "update-output"
    },
    "unfinished": {
      "id": "mechanism-unfinished",
      "kind": "edge",
      "label": "unfinished"
    },
    "finished": {
      "id": "mechanism-finished",
      "kind": "edge",
      "label": "completed"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "continue-then-finish",
      "title": "One continuation followed by completion",
      "description": "The request continues one time, then is completed.",
      "initial": {
        "stage": "Prepared",
        "artifact": "Inputs kept"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Select runnable work",
          "description": "The scheduler selects runnable work.",
          "duration": 3000,
          "active": [
            "state",
            "schedule"
          ],
          "paths": [
            "admit"
          ],
          "state": {
            "stage": "Select runnable work",
            "artifact": "Work admitted under scheduler budgets"
          }
        },
        {
          "title": "Prepare KV capacity",
          "description": "The engine prepares KV capacity.",
          "duration": 3000,
          "active": [
            "kv"
          ],
          "paths": [
            "allocate"
          ],
          "state": {
            "stage": "Prepare KV capacity",
            "artifact": "Allocated or reused blocks and valid mappings"
          }
        },
        {
          "title": "Execute the selected batch",
          "description": "The model executor executes the selected batch.",
          "duration": 3000,
          "active": [
            "execute"
          ],
          "paths": [
            "run"
          ],
          "state": {
            "stage": "Execute the selected batch",
            "artifact": "Model logits"
          }
        },
        {
          "title": "Sample and stream IDs",
          "description": "The engine samples and streams IDs.",
          "duration": 3000,
          "active": [
            "sample"
          ],
          "paths": [
            "sample-output"
          ],
          "state": {
            "stage": "Sample and stream IDs",
            "artifact": "New token sent"
          }
        },
        {
          "title": "Add ID and do a completion check",
          "description": "The engine adds the ID at the end of the sequence. Then it does a completion check.",
          "duration": 3000,
          "active": [
            "update"
          ],
          "paths": [
            "update-output"
          ],
          "state": {
            "stage": "Add ID and do a completion check",
            "artifact": "The constructed request stays unfinished."
          }
        },
        {
          "title": "Return to scheduling",
          "description": "The unfinished request returns to scheduling.",
          "duration": 3000,
          "active": [
            "schedule"
          ],
          "paths": [
            "unfinished"
          ],
          "state": {
            "stage": "Return to scheduling",
            "artifact": "The next step becomes schedulable."
          }
        },
        {
          "title": "Prepare the next KV mapping",
          "description": "The engine prepares the next KV mapping.",
          "duration": 3000,
          "active": [
            "kv"
          ],
          "paths": [
            "allocate"
          ],
          "state": {
            "stage": "Prepare the next KV mapping",
            "artifact": "The next execution has a valid mapping."
          }
        },
        {
          "title": "Execute the next selected batch",
          "description": "The model executor executes the next selected batch.",
          "duration": 3000,
          "active": [
            "execute"
          ],
          "paths": [
            "run"
          ],
          "state": {
            "stage": "Execute the next selected batch",
            "artifact": "The next execution gives logits."
          }
        },
        {
          "title": "Sample the last example token",
          "description": "The engine samples the last example token.",
          "duration": 3000,
          "active": [
            "sample"
          ],
          "paths": [
            "sample-output"
          ],
          "state": {
            "stage": "Sample the last example token",
            "artifact": "The token reaches the constructed stop condition."
          }
        },
        {
          "title": "Set request status to completed",
          "description": "The engine sets the request status to completed.",
          "duration": 3000,
          "active": [
            "update"
          ],
          "paths": [
            "update-output"
          ],
          "state": {
            "stage": "Set request status to completed",
            "artifact": "No more execution for this request"
          }
        },
        {
          "title": "Release this request’s references",
          "description": "The engine releases this request’s references.",
          "duration": 3000,
          "active": [
            "release"
          ],
          "paths": [
            "finished"
          ],
          "state": {
            "stage": "Release this request’s references",
            "artifact": "It is not necessary to free the shared or reusable physical blocks."
          }
        }
      ]
    }
  ]
}