{
  "version": 1,
  "language": "en",
  "title": "Keep alignment of camera inputs with hand inputs",
  "description": "The visual branch uses channel concatenation of skeleton and video latents. The generator adds head and hand embeddings before causal output.",
  "caption": "Author-redrawn Generated Reality control diagram. The visual branch uses channel concatenation of video and skeleton latents. Camera geometry and 3D hand articulation use different encoders. The generator adds their embeddings before chunk generation. New tracking supplies input at the next chunk boundary. This diagram does not show autoregressive-history internals, decoder detail or original result frames. Geometric control alone is not evidence of correct contact or object state.",
  "alt": "Generated Reality flow: the model uses channel concatenation of video and skeleton latents before patchification, then adds camera and hand embeddings.",
  "sources": [
    "https://arxiv.org/abs/2602.18422",
    "https://codeysun.github.io/generated-reality/"
  ],
  "limitations": [
    "The playback is an example. Animation time is not measured runtime.",
    "Chunk names are examples. They are not measured timestamps or tracking-to-display latency.",
    "The three paths show dependencies. They make no claim of encoder execution at the same time."
  ],
  "targets": {
    "head": {
      "id": "mechanism-head",
      "kind": "node",
      "label": "Head pose"
    },
    "camera-encoder": {
      "id": "mechanism-camera-encoder",
      "kind": "node",
      "label": "Camera encoder"
    },
    "hand": {
      "id": "mechanism-hand",
      "kind": "node",
      "label": "Hand pose"
    },
    "motion-encoder": {
      "id": "mechanism-motion-encoder",
      "kind": "node",
      "label": "Motion encoder"
    },
    "skeleton": {
      "id": "mechanism-skeleton",
      "kind": "node",
      "label": "2D skeleton"
    },
    "skeleton-encoder": {
      "id": "mechanism-skeleton-encoder",
      "kind": "node",
      "label": "3D VAE"
    },
    "dit": {
      "id": "mechanism-dit",
      "kind": "node",
      "label": "Token fusion + DiT"
    },
    "frames": {
      "id": "mechanism-frames",
      "kind": "node",
      "label": "Output frames"
    },
    "encode-camera": {
      "id": "mechanism-encode-camera",
      "kind": "edge",
      "label": "encode-camera"
    },
    "encode-hand": {
      "id": "mechanism-encode-hand",
      "kind": "edge",
      "label": "encode-hand"
    },
    "encode-skeleton": {
      "id": "mechanism-encode-skeleton",
      "kind": "edge",
      "label": "encode-skeleton"
    },
    "camera-tokens": {
      "id": "mechanism-camera-tokens",
      "kind": "edge",
      "label": "camera-tokens"
    },
    "hand-tokens": {
      "id": "mechanism-hand-tokens",
      "kind": "edge",
      "label": "hand-tokens"
    },
    "skeleton-tokens": {
      "id": "mechanism-skeleton-tokens",
      "kind": "edge",
      "label": "skeleton-tokens"
    },
    "emit-frames": {
      "id": "mechanism-emit-frames",
      "kind": "edge",
      "label": "emit-frames"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact",
    "chunk": "Chunk"
  },
  "scenarios": [
    {
      "id": "chunk-control",
      "title": "Aligned controls for two chunks",
      "description": "At the two constructed chunk boundaries, camera input, metric hand input and projected hand input keep their identities.",
      "initial": {
        "stage": "Tracking available",
        "artifact": "Chunk k controls",
        "chunk": "k"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Calculate camera features",
          "description": "The camera encoder calculates features from current head pose.",
          "duration": 3000,
          "active": [
            "camera-encoder"
          ],
          "paths": [
            "encode-camera"
          ],
          "state": {
            "stage": "Camera encoding",
            "artifact": "Chunk k camera tokens",
            "chunk": "k"
          }
        },
        {
          "title": "Calculate hand features",
          "description": "The motion encoder and VAE calculate features from metric hand controls and projected hand controls.",
          "duration": 3000,
          "active": [
            "motion-encoder",
            "skeleton-encoder"
          ],
          "paths": [
            "encode-hand",
            "encode-skeleton"
          ],
          "state": {
            "stage": "Hand encoding",
            "artifact": "Hand tokens in 3D and 2D for chunk k",
            "chunk": "k"
          }
        },
        {
          "title": "Add embeddings for the chunk",
          "description": "The generator adds camera and hand embeddings to the visual tokens for the same chunk.",
          "duration": 3000,
          "active": [
            "dit"
          ],
          "paths": [
            "camera-tokens",
            "hand-tokens",
            "skeleton-tokens"
          ],
          "state": {
            "stage": "Conditioning",
            "artifact": "Camera and hand features aligned to chunk k",
            "chunk": "k"
          }
        },
        {
          "title": "Generate first-person frames",
          "description": "The generator generates and supplies first-person frames.",
          "duration": 3000,
          "active": [
            "frames"
          ],
          "paths": [
            "emit-frames"
          ],
          "state": {
            "stage": "Chunk output",
            "artifact": "First-person frames for chunk k",
            "chunk": "k"
          }
        },
        {
          "title": "Read next-chunk tracking",
          "description": "The system reads tracking for the next chunk.",
          "duration": 3000,
          "active": [
            "head",
            "hand",
            "skeleton"
          ],
          "paths": [],
          "state": {
            "stage": "Next tracking input",
            "artifact": "New head and hand controls",
            "chunk": "k + 1"
          }
        },
        {
          "title": "Calculate features in different branches",
          "description": "The encoders calculate features in different branches.",
          "duration": 3000,
          "active": [
            "camera-encoder",
            "motion-encoder",
            "skeleton-encoder"
          ],
          "paths": [
            "encode-camera",
            "encode-hand",
            "encode-skeleton"
          ],
          "state": {
            "stage": "Next encoding",
            "artifact": "Aligned tokens for camera, 3D hand and 2D skeleton",
            "chunk": "k + 1"
          }
        },
        {
          "title": "Supply inputs for the next chunk",
          "description": "The next autoregressive chunk receives the control embeddings.",
          "duration": 3000,
          "active": [
            "dit"
          ],
          "paths": [
            "camera-tokens",
            "hand-tokens",
            "skeleton-tokens"
          ],
          "state": {
            "stage": "Next conditioning",
            "artifact": "Chunk k + 1 embedding addition",
            "chunk": "k + 1"
          }
        },
        {
          "title": "Supply next frames",
          "description": "The generator supplies the next frames.",
          "duration": 3000,
          "active": [
            "frames"
          ],
          "paths": [
            "emit-frames"
          ],
          "state": {
            "stage": "Next chunk output",
            "artifact": "First-person frames for chunk k + 1",
            "chunk": "k + 1"
          }
        }
      ]
    }
  ]
}