{
  "version": 1,
  "language": "en",
  "title": "Dreamer learning and actor deployment",
  "description": "World-model fitting uses replay to infer latent state and fit prediction heads. Behavior learning starts from replay states. Actor and dynamics give imagined trajectories for return estimation and actor/critic updates. A different deployment path infers the environment state and samples the actor without lookahead search.",
  "caption": "Dreamer trains the world model from replay, then trains actor and critic on imagined trajectories that start from replay states. Deployment samples the actor after state inference without lookahead search. Author-redrawn mechanism from Hafner et al., without reconstruction images and detailed loss/gradient paths. Source: Hafner et al. Playback time is illustrative and is not measured runtime.",
  "alt": "Dreamer flow: replay trains dynamics and imagined trajectories train actor/critic, then deployment uses live state inference and the actor.",
  "sources": [
    "https://arxiv.org/abs/2301.04104",
    "https://arxiv.org/html/2301.04104v2"
  ],
  "limitations": [
    "This playback is an example. Playback duration is not measured runtime.",
    "The diagram does not include detailed model/behavior loss gradients and optimizer cadence.",
    "Deployment uses recurrent state inference and actor sampling, not lookahead search."
  ],
  "targets": {
    "replay": {
      "id": "mechanism-replay",
      "kind": "node",
      "label": "Replay"
    },
    "model-state": {
      "id": "mechanism-model-state",
      "kind": "node",
      "label": "Infer latent state"
    },
    "model-fit": {
      "id": "mechanism-model-fit",
      "kind": "node",
      "label": "Model fitting"
    },
    "replay-state": {
      "id": "mechanism-replay-state",
      "kind": "node",
      "label": "Replay state"
    },
    "imagine": {
      "id": "mechanism-imagine",
      "kind": "node",
      "label": "Imagine"
    },
    "returns": {
      "id": "mechanism-returns",
      "kind": "node",
      "label": "Returns"
    },
    "behavior": {
      "id": "mechanism-behavior",
      "kind": "node",
      "label": "Actor + critic"
    },
    "live-observation": {
      "id": "mechanism-live-observation",
      "kind": "node",
      "label": "New observation"
    },
    "live-state": {
      "id": "mechanism-live-state",
      "kind": "node",
      "label": "Infer latent state"
    },
    "live-actor": {
      "id": "mechanism-live-actor",
      "kind": "node",
      "label": "Actor sampling"
    },
    "replay-infer": {
      "id": "mechanism-replay-infer",
      "kind": "edge",
      "label": "infer replay state"
    },
    "fit-heads": {
      "id": "mechanism-fit-heads",
      "kind": "edge",
      "label": "fit model heads"
    },
    "seed-imagination": {
      "id": "mechanism-seed-imagination",
      "kind": "edge",
      "label": "seed imagination"
    },
    "start-imagination": {
      "id": "mechanism-start-imagination",
      "kind": "edge",
      "label": "start rollout"
    },
    "rollout-returns": {
      "id": "mechanism-rollout-returns",
      "kind": "edge",
      "label": "estimate returns"
    },
    "update-behavior": {
      "id": "mechanism-update-behavior",
      "kind": "edge",
      "label": "update actor/critic"
    },
    "policy-update": {
      "id": "mechanism-policy-update",
      "kind": "edge",
      "label": "update policy"
    },
    "live-infer": {
      "id": "mechanism-live-infer",
      "kind": "edge",
      "label": "infer live state"
    },
    "live-act": {
      "id": "mechanism-live-act",
      "kind": "edge",
      "label": "sample action"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "training",
      "title": "Model and behavior learning",
      "description": "This teaching sequence shows selected learning paths. It does not set the global optimizer cadence.",
      "initial": {
        "stage": "Ready",
        "artifact": "Inputs available"
      },
      "unselected": [
        "live-observation",
        "live-state",
        "live-actor",
        "live-infer",
        "live-act"
      ],
      "steps": [
        {
          "title": "Replay input",
          "description": "The world-model training path reads a replay batch of observations and actions.",
          "duration": 3000,
          "active": [
            "replay"
          ],
          "paths": [],
          "state": {
            "stage": "Replay input",
            "artifact": "Replayed observation/action"
          }
        },
        {
          "title": "Posterior-state inference",
          "description": "The model infers a latent posterior state from replay.",
          "duration": 3000,
          "active": [
            "model-state"
          ],
          "paths": [
            "replay-infer"
          ],
          "state": {
            "stage": "Posterior-state inference",
            "artifact": "Latent posterior available"
          }
        },
        {
          "title": "Prediction-head fitting",
          "description": "The model fits reconstruction, reward and continuation heads.",
          "duration": 3000,
          "active": [
            "model-fit"
          ],
          "paths": [
            "fit-heads"
          ],
          "state": {
            "stage": "Prediction-head fitting",
            "artifact": "Reconstruction, reward and continuation"
          }
        },
        {
          "title": "Replay start-state selection",
          "description": "The behavior-learning path selects replay posterior states for rollout initialization.",
          "duration": 3000,
          "active": [
            "replay-state"
          ],
          "paths": [
            "seed-imagination"
          ],
          "state": {
            "stage": "Replay start-state selection",
            "artifact": "Posterior seeds available"
          }
        },
        {
          "title": "Imagined latent rollout",
          "description": "The actor and dynamics make an imagined latent rollout.",
          "duration": 3000,
          "active": [
            "imagine"
          ],
          "paths": [
            "start-imagination"
          ],
          "state": {
            "stage": "Imagined latent rollout",
            "artifact": "The actor and dynamics give an imagined latent rollout."
          }
        },
        {
          "title": "Return estimation",
          "description": "The return calculation uses reward and value.",
          "duration": 3000,
          "active": [
            "returns"
          ],
          "paths": [
            "rollout-returns"
          ],
          "state": {
            "stage": "Return estimation",
            "artifact": "Reward and value are inputs to return calculation."
          }
        },
        {
          "title": "Actor and critic update",
          "description": "The learning process updates the actor and critic parameters.",
          "duration": 3000,
          "active": [
            "behavior"
          ],
          "paths": [
            "update-behavior"
          ],
          "state": {
            "stage": "Actor and critic update",
            "artifact": "Behavior parameters updated"
          }
        },
        {
          "title": "Updated behavior in a rollout",
          "description": "A subsequent imagined rollout can use the behavior update.",
          "duration": 3000,
          "active": [
            "imagine"
          ],
          "paths": [
            "policy-update"
          ],
          "state": {
            "stage": "Updated behavior in a rollout",
            "artifact": "The next imagined rollout can use the update."
          }
        }
      ]
    },
    {
      "id": "deployment",
      "title": "Deployment without search",
      "description": "This trace uses recurrent state inference and actor sampling without lookahead search.",
      "initial": {
        "stage": "Ready",
        "artifact": "Inputs available"
      },
      "unselected": [
        "replay",
        "model-state",
        "model-fit",
        "replay-state",
        "imagine",
        "returns",
        "behavior",
        "replay-infer",
        "fit-heads",
        "seed-imagination",
        "start-imagination",
        "rollout-returns",
        "update-behavior",
        "policy-update"
      ],
      "steps": [
        {
          "title": "Environment observation",
          "description": "The deployment path receives an observation from the environment.",
          "duration": 3000,
          "active": [
            "live-observation"
          ],
          "paths": [],
          "state": {
            "stage": "Environment observation",
            "artifact": "Current environment observation"
          }
        },
        {
          "title": "Recurrent-state inference",
          "description": "The learned model infers recurrent state for deployment.",
          "duration": 3000,
          "active": [
            "live-state"
          ],
          "paths": [
            "live-infer"
          ],
          "state": {
            "stage": "Recurrent-state inference",
            "artifact": "State inferred with the learned model"
          }
        },
        {
          "title": "Actor sampling",
          "description": "The deployed actor samples an action without lookahead search.",
          "duration": 3000,
          "active": [
            "live-actor"
          ],
          "paths": [
            "live-act"
          ],
          "state": {
            "stage": "Actor sampling",
            "artifact": "Actor action. No lookahead search"
          }
        }
      ]
    }
  ]
}