{
  "version": 1,
  "language": "en",
  "title": "Search quantities in MuZero",
  "description": "h encodes observation history. Search queries f for policy and value, and g for candidate next state and reward. A selected action enters environment execution. The new observations then enter the history. A g expansion sends its reward to search and its next state to f.",
  "caption": "MuZero uses h for history encoding, g for latent transitions and rewards, and f for policy/value predictions in search. After environment actions, h encodes new observations. Author-redrawn mechanism from Schrittwieser et al. The source figure keeps the original board snapshots, full search tree and training-unroll panel. Source: Schrittwieser et al. Playback time is illustrative and is not measured runtime.",
  "alt": "MuZero search: history encoding supplies latent dynamics, reward, policy and value predictions before action and new observations.",
  "sources": [
    "https://arxiv.org/abs/1911.08265"
  ],
  "limitations": [
    "This playback is an example. Playback duration is not measured runtime.",
    "The diagram does not include search-visit statistics, full selection/backup rules and training unroll.",
    "This expansion does not show a fixed search depth or measured runtime."
  ],
  "targets": {
    "history": {
      "id": "mechanism-history",
      "kind": "node",
      "label": "History"
    },
    "encode-h": {
      "id": "mechanism-encode-h",
      "kind": "node",
      "label": "Encoder h"
    },
    "search": {
      "id": "mechanism-search",
      "kind": "node",
      "label": "Latent search"
    },
    "act": {
      "id": "mechanism-act",
      "kind": "node",
      "label": "Execute action"
    },
    "dynamics-g": {
      "id": "mechanism-dynamics-g",
      "kind": "node",
      "label": "Dynamics g"
    },
    "prediction-f": {
      "id": "mechanism-prediction-f",
      "kind": "node",
      "label": "Prediction f"
    },
    "history-h": {
      "id": "mechanism-history-h",
      "kind": "edge",
      "label": "encode history"
    },
    "h-search": {
      "id": "mechanism-h-search",
      "kind": "edge",
      "label": "initialize search"
    },
    "search-act": {
      "id": "mechanism-search-act",
      "kind": "edge",
      "label": "do selected action"
    },
    "reobserve": {
      "id": "mechanism-reobserve",
      "kind": "edge",
      "label": "read observations"
    },
    "search-g": {
      "id": "mechanism-search-g",
      "kind": "edge",
      "label": "expand candidate"
    },
    "g-reward": {
      "id": "mechanism-g-reward",
      "kind": "edge",
      "label": "return reward"
    },
    "g-next-f": {
      "id": "mechanism-g-next-f",
      "kind": "edge",
      "label": "supply next state"
    },
    "search-f": {
      "id": "mechanism-search-f",
      "kind": "edge",
      "label": "query policy/value"
    },
    "f-search": {
      "id": "mechanism-f-search",
      "kind": "edge",
      "label": "return policy/value"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "search-expansion",
      "title": "One search-expansion example",
      "description": "This trace shows one expansion. It does not give a fixed search depth.",
      "initial": {
        "stage": "Ready",
        "artifact": "Inputs available"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Observation-history input",
          "description": "The system reads observation history from the environment.",
          "duration": 3000,
          "active": [
            "history"
          ],
          "paths": [],
          "state": {
            "stage": "Observation-history input",
            "artifact": "Environment history available"
          }
        },
        {
          "title": "Root-state encoding",
          "description": "Function h encodes observation history as the root latent state.",
          "duration": 3000,
          "active": [
            "encode-h"
          ],
          "paths": [
            "history-h"
          ],
          "state": {
            "stage": "Root-state encoding",
            "artifact": "Initial latent state"
          }
        },
        {
          "title": "Latent-search initialization",
          "description": "Search starts from that latent state.",
          "duration": 3000,
          "active": [
            "search"
          ],
          "paths": [
            "h-search"
          ],
          "state": {
            "stage": "Latent-search initialization",
            "artifact": "Search rooted at the latent state"
          }
        },
        {
          "title": "Root prediction query",
          "description": "Search queries f for the root policy and value.",
          "duration": 3000,
          "active": [
            "prediction-f"
          ],
          "paths": [
            "search-f"
          ],
          "state": {
            "stage": "Root prediction query",
            "artifact": "f evaluates the root state."
          }
        },
        {
          "title": "Root policy and value",
          "description": "f returns the root policy prior and value to search.",
          "duration": 3000,
          "active": [
            "search"
          ],
          "paths": [
            "f-search"
          ],
          "state": {
            "stage": "Root policy and value",
            "artifact": "Root search quantities available"
          }
        },
        {
          "title": "Candidate-action evaluation",
          "description": "g predicts a candidate next state and reward.",
          "duration": 3000,
          "active": [
            "dynamics-g"
          ],
          "paths": [
            "search-g"
          ],
          "state": {
            "stage": "Candidate-action evaluation",
            "artifact": "g predicts next state and reward."
          }
        },
        {
          "title": "State and reward transfer",
          "description": "g sends next state to f and reward to search.",
          "duration": 3000,
          "active": [
            "prediction-f",
            "search"
          ],
          "paths": [
            "g-next-f",
            "g-reward"
          ],
          "state": {
            "stage": "State and reward transfer",
            "artifact": "Next-state query plus candidate reward"
          }
        },
        {
          "title": "Candidate policy and value",
          "description": "f returns predicted policy and value for the candidate.",
          "duration": 3000,
          "active": [
            "search"
          ],
          "paths": [
            "f-search"
          ],
          "state": {
            "stage": "Candidate policy and value",
            "artifact": "Search can compare the candidate."
          }
        },
        {
          "title": "Selected action execution",
          "description": "The environment executes the action selected by search.",
          "duration": 3000,
          "active": [
            "act"
          ],
          "paths": [
            "search-act"
          ],
          "state": {
            "stage": "Selected action execution",
            "artifact": "Environment action executed"
          }
        },
        {
          "title": "New observation",
          "description": "The system reads new observations. h must encode the new history before the next search.",
          "duration": 3000,
          "active": [
            "history"
          ],
          "paths": [
            "reobserve"
          ],
          "state": {
            "stage": "New observation",
            "artifact": "New history for encoding before the next search"
          }
        }
      ]
    }
  ]
}