{
  "version": 1,
  "language": "en",
  "title": "EgoEngine: aligned robot views and actions",
  "description": "The two synthesis branches use the same digital twin. The optimized action trajectory controls robot rendering. The two outputs have the same object path and time index.",
  "caption": "The author made this diagram from the source figure. A shared digital twin and optimized trajectory supply aligned pairs of synthesized robot observations and executable actions. The solver can use Replay, MPC or residual RL when necessary. This is not a claim of full MCTS. The diagram does not show source input videos, generated frames or deployment results.",
  "alt": "The data flow uses the same digital twin and optimized robot trajectory for action synthesis and robot-view rendering to make aligned training pairs.",
  "sources": [
    "https://arxiv.org/abs/2606.12604",
    "https://egoengine.github.io/"
  ],
  "limitations": [
    "The playback is an example. Playback time is not measured runtime."
  ],
  "targets": {
    "video": {
      "id": "mechanism-video",
      "kind": "node",
      "label": "Human video"
    },
    "twin": {
      "id": "mechanism-twin",
      "kind": "node",
      "label": "Digital twin"
    },
    "visual": {
      "id": "mechanism-visual",
      "kind": "node",
      "label": "Observation synthesis"
    },
    "action": {
      "id": "mechanism-action",
      "kind": "node",
      "label": "Action synthesis"
    },
    "paired": {
      "id": "mechanism-paired",
      "kind": "node",
      "label": "Aligned dataset"
    },
    "reconstruct": {
      "id": "mechanism-reconstruct",
      "kind": "edge",
      "label": "reconstruct"
    },
    "twin-visual": {
      "id": "mechanism-twin-visual",
      "kind": "edge",
      "label": "Supply the scene"
    },
    "twin-action": {
      "id": "mechanism-twin-action",
      "kind": "edge",
      "label": "Supply the scene"
    },
    "render-trajectory": {
      "id": "mechanism-render-trajectory",
      "kind": "edge",
      "label": "Render the trajectory"
    },
    "observation-pair": {
      "id": "mechanism-observation-pair",
      "kind": "edge",
      "label": "Supply observations"
    },
    "action-pair": {
      "id": "mechanism-action-pair",
      "kind": "edge",
      "label": "Supply actions"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "aligned-pair",
      "title": "Make an aligned pair",
      "description": "Make an aligned pair.",
      "initial": {
        "stage": "Start",
        "artifact": "Available inputs"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Reconstruct the same scene",
          "description": "Reconstruct the same scene.",
          "duration": 3000,
          "active": [
            "twin"
          ],
          "paths": [
            "reconstruct"
          ],
          "state": {
            "stage": "Reconstruction",
            "artifact": "Object-centric digital twin and camera geometry"
          }
        },
        {
          "title": "Action retargeting and refinement",
          "description": "Retarget the action. Refine it.",
          "duration": 3000,
          "active": [
            "action"
          ],
          "paths": [
            "twin-action"
          ],
          "state": {
            "stage": "Action generation",
            "artifact": "Object-motion-matched trajectory / replay, MPC or residual RL as necessary"
          }
        },
        {
          "title": "Supply the scene and optimized trajectory",
          "description": "Supply the scene and optimized trajectory.",
          "duration": 3000,
          "active": [
            "visual"
          ],
          "paths": [
            "twin-visual",
            "render-trajectory"
          ],
          "state": {
            "stage": "Observation inputs",
            "artifact": "Scene geometry plus the same optimized robot motion"
          }
        },
        {
          "title": "Make the aligned robot view",
          "description": "Make the aligned robot view.",
          "duration": 3000,
          "active": [
            "visual"
          ],
          "paths": [],
          "state": {
            "stage": "Observation generation",
            "artifact": "Human removal, robot rendering and occlusion-aware blending"
          }
        },
        {
          "title": "Make pairs from the two branches",
          "description": "Make pairs from the outputs of the two branches.",
          "duration": 3000,
          "active": [
            "paired"
          ],
          "paths": [
            "observation-pair",
            "action-pair"
          ],
          "state": {
            "stage": "Paired supervision",
            "artifact": "Robot observation and executable action at the same time index"
          }
        }
      ]
    }
  ]
}