{
  "version": 1,
  "language": "en",
  "title": "DreamDojo: pretrain, adapt and distill",
  "description": "For world-model pretraining, DreamDojo calculates latent proxy actions from human frame pairs. During target-robot adaptation, DreamDojo resets the action conditioner and grounds controls with target-robot data. Distillation then trains an autoregressive student.",
  "caption": "The author made this diagram from the source figure. Human transitions supply proxy actions. Target-robot adaptation grounds the command interface before autoregressive distillation. The training phases are different. The playback is an example. It does not show measured training time or generated-video performance.",
  "alt": "The training flow uses human-video latent actions for pretraining, robot data for action grounding, then autoregressive distillation.",
  "sources": [
    "https://arxiv.org/abs/2602.06949",
    "https://dreamdojo-world.github.io/"
  ],
  "limitations": [
    "The playback is an example. Playback time is not measured runtime.",
    "The diagram does not show robot photographs, generated frames or measured task performance."
  ],
  "targets": {
    "frames": {
      "id": "mechanism-frames",
      "kind": "node",
      "label": "Human frame pairs"
    },
    "latent": {
      "id": "mechanism-latent",
      "kind": "node",
      "label": "Calculate latent actions"
    },
    "prior": {
      "id": "mechanism-prior",
      "kind": "node",
      "label": "Video world model"
    },
    "robot-data": {
      "id": "mechanism-robot-data",
      "kind": "node",
      "label": "Target robot data"
    },
    "conditioner": {
      "id": "mechanism-conditioner",
      "kind": "node",
      "label": "Reset the conditioner"
    },
    "grounded": {
      "id": "mechanism-grounded",
      "kind": "node",
      "label": "Robot world model"
    },
    "distill": {
      "id": "mechanism-distill",
      "kind": "node",
      "label": "Distill the student"
    },
    "student": {
      "id": "mechanism-student",
      "kind": "node",
      "label": "Distilled student"
    },
    "infer": {
      "id": "mechanism-infer",
      "kind": "edge",
      "label": "Calculate latent actions"
    },
    "pretrain": {
      "id": "mechanism-pretrain",
      "kind": "edge",
      "label": "pretrain"
    },
    "condition": {
      "id": "mechanism-condition",
      "kind": "edge",
      "label": "Supply robot data"
    },
    "ground": {
      "id": "mechanism-ground",
      "kind": "edge",
      "label": "ground"
    },
    "reuse-prior": {
      "id": "mechanism-reuse-prior",
      "kind": "edge",
      "label": "Use the prior again"
    },
    "teacher": {
      "id": "mechanism-teacher",
      "kind": "edge",
      "label": "Supply the teacher"
    },
    "distilled": {
      "id": "mechanism-distilled",
      "kind": "edge",
      "label": "Supply the student"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "training-stages",
      "title": "Three training phases",
      "description": "The example has three training phases.",
      "initial": {
        "stage": "Start",
        "artifact": "Available inputs"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Start from human transitions",
          "description": "Start from human transitions.",
          "duration": 3000,
          "active": [
            "frames"
          ],
          "paths": [],
          "state": {
            "stage": "Human-video data",
            "artifact": "Unlabeled frame transitions"
          }
        },
        {
          "title": "Calculate the proxy action",
          "description": "Calculate the proxy action.",
          "duration": 3000,
          "active": [
            "latent"
          ],
          "paths": [
            "infer"
          ],
          "state": {
            "stage": "Latent inference",
            "artifact": "32D transition code with continuous values"
          }
        },
        {
          "title": "Pretrain the world model",
          "description": "Pretrain the world model.",
          "duration": 3000,
          "active": [
            "prior"
          ],
          "paths": [
            "pretrain"
          ],
          "state": {
            "stage": "Pretraining",
            "artifact": "Visual dynamics prior with latent conditioning"
          }
        },
        {
          "title": "Learn the target command interface",
          "description": "Learn the target command interface.",
          "duration": 3000,
          "active": [
            "conditioner"
          ],
          "paths": [
            "condition"
          ],
          "state": {
            "stage": "Robot adaptation",
            "artifact": "The reset conditioner learns target-robot commands."
          }
        },
        {
          "title": "Prior reuse and control grounding",
          "description": "Keep the prior. Ground the controls.",
          "duration": 3000,
          "active": [
            "grounded"
          ],
          "paths": [
            "reuse-prior",
            "ground"
          ],
          "state": {
            "stage": "Grounded model",
            "artifact": "Pretrained interaction knowledge plus meanings of the target robot commands"
          }
        },
        {
          "title": "Use the grounded teacher",
          "description": "Use the grounded teacher.",
          "duration": 3000,
          "active": [
            "distill"
          ],
          "paths": [
            "teacher"
          ],
          "state": {
            "stage": "Distillation",
            "artifact": "Autoregressive student training"
          }
        },
        {
          "title": "Supply the student",
          "description": "Supply the student.",
          "duration": 3000,
          "active": [
            "student"
          ],
          "paths": [
            "distilled"
          ],
          "state": {
            "stage": "Interactive model",
            "artifact": "Distilled predictor for subsequent operation"
          }
        }
      ]
    }
  ]
}