{
  "version": 1,
  "language": "en",
  "title": "Different roles for observation, action and prediction",
  "description": "The action/observation trace and prediction/memory trace show different roles. They do not give a new implementation or a schedule for all implementations.",
  "caption": "Author-redrawn explanation of selected connections in the modular architecture in Xun Huang’s blog and Yann LeCun’s slides. The diagram does not show the configurator or auxiliary connections. This is a role diagram. It is not a new implementation or evidence that modularity makes scaling better. Source: Xun Huang's blog and Yann LeCun's slides. The playback is an example. Animation time is not measured runtime.",
  "alt": "Role flow: perception supplies input to the actor and world model. Actions change the environment. Memory helps predictions sent to cost modules.",
  "sources": [
    "https://www.xunhuang.me/blogs/world_model.html",
    "https://leshouches2022.github.io/SLIDES/compressed-yann-1.pdf"
  ],
  "limitations": [
    "The execution is an example. Playback time is not measured runtime.",
    "The diagram does not show the configurator or auxiliary links.",
    "The diagram has no cost-to-actor edge. Cost evaluation is not a prerequisite for action selection in this diagram."
  ],
  "targets": {
    "perception": {
      "id": "mechanism-perception",
      "kind": "node",
      "label": "Perception"
    },
    "world": {
      "id": "mechanism-world",
      "kind": "node",
      "label": "World model"
    },
    "cost": {
      "id": "mechanism-cost",
      "kind": "node",
      "label": "Cost"
    },
    "memory": {
      "id": "mechanism-memory",
      "kind": "node",
      "label": "Memory"
    },
    "actor": {
      "id": "mechanism-actor",
      "kind": "node",
      "label": "Actor"
    },
    "environment": {
      "id": "mechanism-environment",
      "kind": "node",
      "label": "Environment"
    },
    "perception-world": {
      "id": "mechanism-perception-world",
      "kind": "edge",
      "label": "State for prediction"
    },
    "perception-actor": {
      "id": "mechanism-perception-actor",
      "kind": "edge",
      "label": "State for action"
    },
    "actor-world": {
      "id": "mechanism-actor-world",
      "kind": "edge",
      "label": "Candidate action"
    },
    "world-cost": {
      "id": "mechanism-world-cost",
      "kind": "edge",
      "label": "Predicted consequences"
    },
    "memory-read": {
      "id": "mechanism-memory-read",
      "kind": "edge",
      "label": "Read short-term state"
    },
    "memory-write": {
      "id": "mechanism-memory-write",
      "kind": "edge",
      "label": "Update short-term state"
    },
    "actor-environment": {
      "id": "mechanism-actor-environment",
      "kind": "edge",
      "label": "Execute action"
    },
    "environment-perception": {
      "id": "mechanism-environment-perception",
      "kind": "edge",
      "label": "Observe environment"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "act-observe",
      "title": "Action and observation",
      "description": "This trace shows selected action/perception links. It adds no cost or prediction prerequisite for execution.",
      "initial": {
        "stage": "Ready",
        "artifact": "Inputs are available."
      },
      "unselected": [
        "world",
        "cost",
        "memory",
        "perception-world",
        "actor-world",
        "world-cost",
        "memory-read",
        "memory-write"
      ],
      "steps": [
        {
          "title": "Encode the observation",
          "description": "Encode the observation",
          "duration": 3000,
          "active": [
            "perception"
          ],
          "paths": [],
          "state": {
            "stage": "Encode the observation",
            "artifact": "Visual state is available."
          }
        },
        {
          "title": "Supply state to the actor",
          "description": "Supply state to the actor",
          "duration": 3000,
          "active": [
            "actor"
          ],
          "paths": [
            "perception-actor"
          ],
          "state": {
            "stage": "Supply state to the actor",
            "artifact": "Actor receives visual state."
          }
        },
        {
          "title": "Execute an action",
          "description": "Execute an action",
          "duration": 3000,
          "active": [
            "environment"
          ],
          "paths": [
            "actor-environment"
          ],
          "state": {
            "stage": "Execute an action",
            "artifact": "Environment receives an action."
          }
        },
        {
          "title": "Observe the changed scene",
          "description": "Observe the changed scene",
          "duration": 3000,
          "active": [
            "perception"
          ],
          "paths": [
            "environment-perception"
          ],
          "state": {
            "stage": "Observe the changed scene",
            "artifact": "A new percept is available."
          }
        }
      ]
    },
    {
      "id": "predict-memory",
      "title": "Prediction and memory",
      "description": "This is one prediction trace. It gives no universal scheduling barrier or cost-to-actor edge.",
      "initial": {
        "stage": "Ready",
        "artifact": "Inputs are available."
      },
      "unselected": [
        "environment",
        "actor-environment",
        "environment-perception",
        "perception-actor"
      ],
      "steps": [
        {
          "title": "Supply observation and action",
          "description": "Supply observation and action",
          "duration": 3000,
          "active": [
            "perception",
            "actor"
          ],
          "paths": [],
          "state": {
            "stage": "Supply observation and action",
            "artifact": "State and candidate action are available."
          }
        },
        {
          "title": "Condition the prediction",
          "description": "Condition the prediction",
          "duration": 3000,
          "active": [
            "world"
          ],
          "paths": [
            "perception-world",
            "actor-world"
          ],
          "state": {
            "stage": "Condition the prediction",
            "artifact": "World model receives state and action."
          }
        },
        {
          "title": "Read short-term state",
          "description": "Read short-term state",
          "duration": 3000,
          "active": [
            "world"
          ],
          "paths": [
            "memory-read"
          ],
          "state": {
            "stage": "Read short-term state",
            "artifact": "The world model receives short-term state."
          }
        },
        {
          "title": "Predict consequences",
          "description": "Predict consequences",
          "duration": 3000,
          "active": [
            "world"
          ],
          "paths": [],
          "state": {
            "stage": "Predict consequences",
            "artifact": "Predicted state is available."
          }
        },
        {
          "title": "Give prediction and update memory",
          "description": "The world model gives its prediction to the cost module and updates memory.",
          "duration": 3000,
          "active": [
            "cost",
            "memory"
          ],
          "paths": [
            "world-cost",
            "memory-write"
          ],
          "state": {
            "stage": "Give prediction and update memory",
            "artifact": "Cost input and short-term state are available."
          }
        }
      ]
    }
  ]
}