{
  "version": 1,
  "language": "en",
  "title": "Video2Sim2Real: repair keyframes and correct contact",
  "description": "One RGB-D demonstration supplies a digital twin and motion prior for the repair of contact, interaction and release keyframes. At deployment, perception calibration corrects poses from an object point cloud before entry. Local residual control then corrects contact dynamics.",
  "caption": "The author made this diagram from the source figure. Object-event anchors repair the offline reference. Perception calibration corrects geometric placement before entry. Residual control corrects local contact dynamics. The diagram does not show the optional motion planner or source robot-result images. It does not show a measured rollout.",
  "alt": "The skill flow repairs keyframes in a digital twin, then uses perception calibration before entry and residual control during contact.",
  "sources": [
    "https://arxiv.org/abs/2606.08828",
    "https://video2sim2real.github.io/"
  ],
  "limitations": [
    "The playback is an example. Playback time is not measured runtime."
  ],
  "targets": {
    "demo": {
      "id": "mechanism-demo",
      "kind": "node",
      "label": "RGB-D video"
    },
    "twin": {
      "id": "mechanism-twin",
      "kind": "node",
      "label": "Twin + motion prior"
    },
    "repair": {
      "id": "mechanism-repair",
      "kind": "node",
      "label": "Repair keyframes"
    },
    "cloud": {
      "id": "mechanism-cloud",
      "kind": "node",
      "label": "Object cloud"
    },
    "calibration": {
      "id": "mechanism-calibration",
      "kind": "node",
      "label": "Perception calibration"
    },
    "residual": {
      "id": "mechanism-residual",
      "kind": "node",
      "label": "Use base + residual"
    },
    "reconstruct": {
      "id": "mechanism-reconstruct",
      "kind": "edge",
      "label": "reconstruct"
    },
    "repair-reference": {
      "id": "mechanism-repair-reference",
      "kind": "edge",
      "label": "Repair keyframes"
    },
    "reference-to-il": {
      "id": "mechanism-reference-to-il",
      "kind": "edge",
      "label": "Supply the reference"
    },
    "perceive": {
      "id": "mechanism-perceive",
      "kind": "edge",
      "label": "Use the object cloud"
    },
    "local-control": {
      "id": "mechanism-local-control",
      "kind": "edge",
      "label": "Correct contact"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "build-and-deploy",
      "title": "Skill construction and operation",
      "description": "Make the skill. Use it.",
      "initial": {
        "stage": "Start",
        "artifact": "Available inputs"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Reconstruct the single-video problem",
          "description": "Reconstruct the problem from one video.",
          "duration": 3000,
          "active": [
            "twin"
          ],
          "paths": [
            "reconstruct"
          ],
          "state": {
            "stage": "Offline reconstruction",
            "artifact": "Scene assets and an initial retargeted trajectory for the robot"
          }
        },
        {
          "title": "Repair the causal event anchors",
          "description": "Repair the causal event anchors.",
          "duration": 3000,
          "active": [
            "repair"
          ],
          "paths": [
            "repair-reference"
          ],
          "state": {
            "stage": "Offline refinement",
            "artifact": "Contact, interaction and release constraints repair the reference."
          }
        },
        {
          "title": "Supply the refined reference",
          "description": "Supply the refined reference.",
          "duration": 3000,
          "active": [
            "calibration"
          ],
          "paths": [
            "reference-to-il"
          ],
          "state": {
            "stage": "Deployment inputs",
            "artifact": "Calibration can use the refined keyframe geometry."
          }
        },
        {
          "title": "Correct the keyframe poses",
          "description": "Correct keyframe poses from the current object cloud.",
          "duration": 3000,
          "active": [
            "calibration"
          ],
          "paths": [
            "perceive"
          ],
          "state": {
            "stage": "Before entry",
            "artifact": "The imitation model corrects keyframe poses from current perception."
          }
        },
        {
          "title": "Execute with local residual correction",
          "description": "Execute the skill with local residual correction.",
          "duration": 3000,
          "active": [
            "residual"
          ],
          "paths": [
            "local-control"
          ],
          "state": {
            "stage": "During contact",
            "artifact": "Geometric base trajectory plus finger-level residual control"
          }
        }
      ]
    }
  ]
}