{
  "version": 1,
  "language": "en",
  "title": "One RL step hands weights to the rollout engine",
  "description": "Two source-derived traces follow one RL training step. In verl's colocated sync mode, the engine and trainer take turns on the same GPUs. In AReaL's disaggregated mode, training overlaps generation and new weights interrupt running requests.",
  "caption": "Author-created trace from verl v0.9.1 (HYBRID mode, vLLM rollout) and AReaL v2.1.0 (xccl weight updates). Function names follow the pinned source. Playback time is not measured latency or memory use.",
  "alt": "Flow diagram: an RL controller sends prompts to a rollout engine, the trainer learns from buffered trajectories, and new weights return to the engine",
  "stateLabels": {
    "placement": "Placement",
    "engine": "Engine GPU memory",
    "trainer": "Trainer",
    "version": "Policy version",
    "transport": "Weight transport",
    "staleness": "Staleness rule"
  },
  "targets": {
    "controller": {
      "id": "rl-controller",
      "kind": "node",
      "label": "Controller"
    },
    "engine": {
      "id": "rl-engine",
      "kind": "node",
      "label": "Rollout engine"
    },
    "buffer": {
      "id": "rl-buffer",
      "kind": "node",
      "label": "Replay buffer"
    },
    "trainer": {
      "id": "rl-trainer",
      "kind": "node",
      "label": "Trainer"
    },
    "prompts": {
      "id": "rl-prompts",
      "kind": "edge",
      "label": "Send prompts"
    },
    "release": {
      "id": "rl-release",
      "kind": "edge",
      "label": "Sleep, wake or pause the engine"
    },
    "trajectories": {
      "id": "rl-trajectories",
      "kind": "edge",
      "label": "Write trajectories"
    },
    "batch": {
      "id": "rl-batch",
      "kind": "edge",
      "label": "Read a training batch"
    },
    "weights": {
      "id": "rl-weights",
      "kind": "edge",
      "label": "Send new weights"
    }
  },
  "sources": [
    {
      "title": "Original author diagram",
      "file": "diagram.svg",
      "sha256": "69ea731a4a29f092d19a6ac269f4a1374c2dacbf6b9f2bb52d12441c09072e1e"
    },
    {
      "title": "verl v0.9.1 synchronous trainer",
      "url": "https://github.com/verl-project/verl/blob/v0.9.1/verl/trainer/ppo/v1/trainer_sync.py"
    },
    {
      "title": "verl v0.9.1 colocated weight update",
      "url": "https://github.com/verl-project/verl/blob/v0.9.1/verl/workers/engine_workers.py"
    },
    {
      "title": "verl v0.9.1 vLLM rollout server",
      "url": "https://github.com/verl-project/verl/blob/v0.9.1/verl/workers/rollout/vllm_rollout/vllm_async_server.py"
    },
    {
      "title": "AReaL v2.1.0 staleness manager",
      "url": "https://github.com/areal-project/AReaL/blob/v2.1.0/areal/infra/staleness_manager.py"
    },
    {
      "title": "AReaL v2.1.0 remote inference engine",
      "url": "https://github.com/areal-project/AReaL/blob/v2.1.0/areal/infra/remote_inf_engine.py"
    },
    {
      "title": "Article evidence and version ledger",
      "file": "../../illustrations/SOURCE.md"
    }
  ],
  "limitations": [
    "Each trace follows one configuration at the pinned tag. verl and AReaL also support other placements and weight transports.",
    "The frameworks pin older engine releases than vLLM v0.32.0 and SGLang v0.5.23. The trace describes the framework code, not the newest engine API.",
    "Steps are a teaching order. Playback duration does not measure latency, memory use or throughput."
  ],
  "scenarios": [
    {
      "id": "verl-colocated-sync",
      "title": "verl HYBRID: take turns on the same GPUs",
      "description": "The vLLM engine and the trainer share GPUs. The engine sleeps during training and receives weights through CUDA IPC before it generates again.",
      "initial": {
        "placement": "same GPUs (HYBRID)",
        "engine": "weights and KV cache",
        "trainer": "offloaded to CPU",
        "version": "v",
        "transport": "none",
        "staleness": "on-policy by construction"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Generate rollouts",
          "description": "The agent loop sends prompts to the vLLM server, a Ray actor that owns AsyncLLM.",
          "duration": 3000,
          "active": [
            "engine"
          ],
          "paths": [
            "prompts"
          ],
          "state": {
            "placement": "same GPUs (HYBRID)",
            "engine": "weights and KV cache",
            "trainer": "offloaded to CPU",
            "version": "v",
            "transport": "none",
            "staleness": "on-policy by construction"
          }
        },
        {
          "title": "Wait for the full batch",
          "description": "Trajectories enter the replay buffer. The sync trainer samples only after the whole batch arrives.",
          "duration": 3000,
          "active": [
            "buffer"
          ],
          "paths": [
            "trajectories"
          ],
          "state": {
            "placement": "same GPUs (HYBRID)",
            "engine": "weights and KV cache",
            "trainer": "offloaded to CPU",
            "version": "v",
            "transport": "none",
            "staleness": "on-policy by construction"
          }
        },
        {
          "title": "Sleep the engine",
          "description": "sleep_replicas() calls AsyncLLM.sleep(level=2). The engine discards its weights and KV cache.",
          "duration": 3000,
          "active": [
            "engine"
          ],
          "paths": [
            "release"
          ],
          "state": {
            "placement": "same GPUs (HYBRID)",
            "engine": "released by level-2 sleep",
            "trainer": "offloaded to CPU",
            "version": "v",
            "transport": "none",
            "staleness": "on-policy by construction"
          }
        },
        {
          "title": "Train the policy",
          "description": "The trainer loads parameters and optimizer state onto the GPUs, then updates the actor.",
          "duration": 3000,
          "active": [
            "trainer"
          ],
          "paths": [
            "batch"
          ],
          "state": {
            "placement": "same GPUs (HYBRID)",
            "engine": "released by level-2 sleep",
            "trainer": "parameters and optimizer on GPU",
            "version": "trainer computes v+1",
            "transport": "none",
            "staleness": "on-policy by construction"
          }
        },
        {
          "title": "Copy weights through CUDA IPC",
          "description": "wake_up(weights) restores empty weight buffers. Weight buckets move through one reusable 2048 MB staging buffer.",
          "duration": 3300,
          "active": [
            "engine"
          ],
          "paths": [
            "weights"
          ],
          "state": {
            "placement": "same GPUs (HYBRID)",
            "engine": "weights only",
            "trainer": "parameters on GPU",
            "version": "v+1",
            "transport": "CUDA IPC buckets",
            "staleness": "on-policy by construction"
          }
        },
        {
          "title": "Hand the GPUs back",
          "description": "The trainer offloads to CPU. wake_up(kv_cache) then creates a fresh KV cache for the next rollout.",
          "duration": 3000,
          "active": [
            "engine"
          ],
          "paths": [
            "release"
          ],
          "state": {
            "placement": "same GPUs (HYBRID)",
            "engine": "weights and fresh KV cache",
            "trainer": "offloaded to CPU",
            "version": "v+1",
            "transport": "CUDA IPC buckets",
            "staleness": "on-policy by construction"
          }
        }
      ]
    },
    {
      "id": "areal-disaggregated-async",
      "title": "AReaL: train while the engines generate",
      "description": "SGLang servers run on their own GPUs. Training overlaps generation, and a weight update interrupts requests that are still running.",
      "initial": {
        "placement": "separate GPUs",
        "engine": "weights and KV cache",
        "trainer": "on its own GPUs",
        "version": "v",
        "transport": "none",
        "staleness": "admission cap, η = 0 by default"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Admit rollouts under the cap",
          "description": "The dispatcher submits while capacity remains under (η + v + 1) × batch size, minus accepted and running work. Each request records version v.",
          "duration": 3600,
          "active": [
            "engine"
          ],
          "paths": [
            "prompts"
          ],
          "state": {
            "placement": "separate GPUs",
            "engine": "weights and KV cache",
            "trainer": "on its own GPUs",
            "version": "v",
            "transport": "none",
            "staleness": "admission cap, η = 0 by default"
          }
        },
        {
          "title": "Collect accepted trajectories",
          "description": "Finished trajectories wait until the trainer takes one batch.",
          "duration": 2600,
          "active": [
            "buffer"
          ],
          "paths": [
            "trajectories"
          ],
          "state": {
            "placement": "separate GPUs",
            "engine": "weights and KV cache",
            "trainer": "on its own GPUs",
            "version": "v",
            "transport": "none",
            "staleness": "admission cap, η = 0 by default"
          }
        },
        {
          "title": "Train while engines generate",
          "description": "The trainer computes step v+1. The engines keep generating with version v while capacity remains.",
          "duration": 3300,
          "active": [
            "trainer",
            "engine"
          ],
          "paths": [
            "batch",
            "prompts"
          ],
          "state": {
            "placement": "separate GPUs",
            "engine": "weights and KV cache",
            "trainer": "computes v+1",
            "version": "engines v, trainer v+1",
            "transport": "none",
            "staleness": "admission cap, η = 0 by default"
          }
        },
        {
          "title": "Pause and interrupt",
          "description": "rollout.pause() stops new admissions. pause_generation() aborts running requests, which return partial output.",
          "duration": 3000,
          "active": [
            "engine"
          ],
          "paths": [
            "release"
          ],
          "state": {
            "placement": "separate GPUs",
            "engine": "paused, running requests aborted",
            "trainer": "finished v+1",
            "version": "engines v, trainer v+1",
            "transport": "none",
            "staleness": "admission cap, η = 0 by default"
          }
        },
        {
          "title": "Broadcast new weights",
          "description": "Trainer rank 0 broadcasts 1024 MB buckets over NCCL into update_weights_from_distributed. Then generation continues.",
          "duration": 3300,
          "active": [
            "engine"
          ],
          "paths": [
            "weights"
          ],
          "state": {
            "placement": "separate GPUs",
            "engine": "weights v+1, no running requests",
            "trainer": "on its own GPUs",
            "version": "v+1",
            "transport": "NCCL broadcast buckets",
            "staleness": "admission cap, η = 0 by default"
          }
        },
        {
          "title": "Resume interrupted requests",
          "description": "Interrupted requests resend the prompt and the tokens generated so far. New tokens record version v+1.",
          "duration": 3300,
          "active": [
            "engine"
          ],
          "paths": [
            "prompts"
          ],
          "state": {
            "placement": "separate GPUs",
            "engine": "weights v+1, requests resumed",
            "trainer": "on its own GPUs",
            "version": "v+1",
            "transport": "NCCL broadcast buckets",
            "staleness": "one trajectory can mix v and v+1"
          }
        }
      ]
    }
  ]
}