{
  "version": 1,
  "language": "en",
  "title": "Genie action codes from video",
  "description": "Video tokenization and latent-action inference are parallel training branches into a video dynamics predictor. In the interaction trace, a selected learned action code replaces inference from video transitions. These codes are not calibrated robot commands.",
  "caption": "Genie trains action-conditioned dynamics with video tokens and latent actions inferred from unlabeled transitions. These latent actions are not calibrated robot commands. Author-redrawn mechanism from Bruce et al., without game footage and generated results. Source: Bruce et al. Playback time is illustrative and is not measured runtime.",
  "alt": "Genie flow: video tokens and inferred latent actions train dynamics, and interaction uses action codes without calibrated robot commands.",
  "sources": [
    "https://arxiv.org/abs/2402.15391"
  ],
  "limitations": [
    "This playback is an example. Playback duration is not measured runtime.",
    "Discrete latent-action IDs are not calibrated robot motor commands.",
    "The interaction trace uses a selected learned code. It does not infer control from a future frame.",
    "This diagram does not include source game footage or generated-result frames."
  ],
  "targets": {
    "video-input": {
      "id": "mechanism-video-input",
      "kind": "node",
      "label": "Video frames"
    },
    "video-tokenizer": {
      "id": "mechanism-video-tokenizer",
      "kind": "node",
      "label": "Tokenization"
    },
    "video-tokens": {
      "id": "mechanism-video-tokens",
      "kind": "node",
      "label": "Video tokens"
    },
    "action-inference": {
      "id": "mechanism-action-inference",
      "kind": "node",
      "label": "Action inference"
    },
    "action-codes": {
      "id": "mechanism-action-codes",
      "kind": "node",
      "label": "Action codes"
    },
    "dynamics": {
      "id": "mechanism-dynamics",
      "kind": "node",
      "label": "Dynamics"
    },
    "next-tokens": {
      "id": "mechanism-next-tokens",
      "kind": "node",
      "label": "Next tokens"
    },
    "video-encode": {
      "id": "mechanism-video-encode",
      "kind": "edge",
      "label": "tokenize video"
    },
    "infer-actions": {
      "id": "mechanism-infer-actions",
      "kind": "edge",
      "label": "infer action codes"
    },
    "video-codes": {
      "id": "mechanism-video-codes",
      "kind": "edge",
      "label": "supply video tokens"
    },
    "latent-codes": {
      "id": "mechanism-latent-codes",
      "kind": "edge",
      "label": "supply action codes"
    },
    "visual-condition": {
      "id": "mechanism-visual-condition",
      "kind": "edge",
      "label": "supply visual tokens"
    },
    "action-condition": {
      "id": "mechanism-action-condition",
      "kind": "edge",
      "label": "supply action code"
    },
    "predict-tokens": {
      "id": "mechanism-predict-tokens",
      "kind": "edge",
      "label": "predict tokens"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "learn-from-video",
      "title": "Learning from unlabeled transitions",
      "description": "This training trace infers latent actions from video transitions.",
      "initial": {
        "stage": "Ready",
        "artifact": "Inputs available"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Training-video input",
          "description": "The training path reads an unlabeled video transition.",
          "duration": 3000,
          "active": [
            "video-input"
          ],
          "paths": [],
          "state": {
            "stage": "Training-video input",
            "artifact": "Unlabeled video transition"
          }
        },
        {
          "title": "Parallel tokenization and inference",
          "description": "The video tokenizer and latent-action inference operate in parallel.",
          "duration": 3000,
          "active": [
            "video-tokenizer",
            "action-inference"
          ],
          "paths": [
            "video-encode",
            "infer-actions"
          ],
          "state": {
            "stage": "Parallel tokenization and inference",
            "artifact": "Two training branches"
          }
        },
        {
          "title": "Video and action codes",
          "description": "The two branches give video tokens and an inferred latent-action code.",
          "duration": 3000,
          "active": [
            "video-tokens",
            "action-codes"
          ],
          "paths": [
            "video-codes",
            "latent-codes"
          ],
          "state": {
            "stage": "Video and action codes",
            "artifact": "Visual tokens plus inferred action code"
          }
        },
        {
          "title": "Dynamics inputs",
          "description": "The dynamics model receives visual history and the latent-action code.",
          "duration": 3000,
          "active": [
            "dynamics"
          ],
          "paths": [
            "visual-condition",
            "action-condition"
          ],
          "state": {
            "stage": "Dynamics inputs",
            "artifact": "History and latent action are available."
          }
        },
        {
          "title": "Next-token prediction",
          "description": "The dynamics model predicts next video tokens.",
          "duration": 3000,
          "active": [
            "next-tokens"
          ],
          "paths": [
            "predict-tokens"
          ],
          "state": {
            "stage": "Next-token prediction",
            "artifact": "Next-token training prediction"
          }
        }
      ]
    },
    {
      "id": "choose-a-code",
      "title": "Interaction with learned codes",
      "description": "This trace uses a selected learned code. It does not supply a calibrated robot action or reproduce a generated-result demonstration.",
      "initial": {
        "stage": "Ready",
        "artifact": "Inputs available"
      },
      "unselected": [
        "action-inference",
        "infer-actions",
        "latent-codes"
      ],
      "steps": [
        {
          "title": "History and selected code",
          "description": "The interaction path starts from visual history and a selected learned code. It does not infer the code from a future transition.",
          "duration": 3000,
          "active": [
            "video-input",
            "action-codes"
          ],
          "paths": [],
          "state": {
            "stage": "History and selected code",
            "artifact": "Selected code. No future-transition inference"
          }
        },
        {
          "title": "History encoding",
          "description": "The video tokenizer encodes the observed visual history.",
          "duration": 3000,
          "active": [
            "video-tokenizer"
          ],
          "paths": [
            "video-encode"
          ],
          "state": {
            "stage": "History encoding",
            "artifact": "Observed video context"
          }
        },
        {
          "title": "Visual tokens",
          "description": "The tokenizer gives tokens for that visual history.",
          "duration": 3000,
          "active": [
            "video-tokens"
          ],
          "paths": [
            "video-codes"
          ],
          "state": {
            "stage": "Visual tokens",
            "artifact": "Visual history tokenized"
          }
        },
        {
          "title": "Selected-code input",
          "description": "The dynamics model receives the history and selected code.",
          "duration": 3000,
          "active": [
            "dynamics"
          ],
          "paths": [
            "visual-condition",
            "action-condition"
          ],
          "state": {
            "stage": "Selected-code input",
            "artifact": "Dynamics receives history and selected code."
          }
        },
        {
          "title": "Predicted continuation",
          "description": "The dynamics model predicts continuation tokens.",
          "duration": 3000,
          "active": [
            "next-tokens"
          ],
          "paths": [
            "predict-tokens"
          ],
          "state": {
            "stage": "Predicted continuation",
            "artifact": "Generated continuation tokens"
          }
        }
      ]
    }
  ]
}