{
  "version": 1,
  "language": "en",
  "title": "Initialize from VLM weights and train on robot data",
  "description": "The training recipe uses visual-language foundation weights and the two robot-data sources to train a generalist policy.",
  "caption": "The author made this diagram from Physical Intelligence’s π0 source figure. Internet-scale vision-language pretraining supplies the backbone. Robot training uses Open X-Embodiment and PI robot data. After this training, the policy can receive prompts or be fine-tuned for tasks. The diagram does not show robot pictures or task icons, and it is not a layer-exact computation graph. The playback is an example, and animation time is not measured runtime.",
  "alt": "Training flow: visual-language foundation weights combine with Open X-Embodiment and PI robot data to train a generalist π0 policy.",
  "sources": [
    "https://www.pi.website/blog/pi0"
  ],
  "limitations": [
    "The playback is an example. Playback time is not measured runtime.",
    "The diagram does not show a robot rollout or performance result.",
    "Dataset arrows show training inputs. They make no claim that all examples have all targets."
  ],
  "targets": {
    "internet": {
      "id": "mechanism-internet",
      "kind": "node",
      "label": "Internet pretraining"
    },
    "foundation": {
      "id": "mechanism-foundation",
      "kind": "node",
      "label": "VLM foundation"
    },
    "openx": {
      "id": "mechanism-openx",
      "kind": "node",
      "label": "Open X-Embodiment"
    },
    "pi-data": {
      "id": "mechanism-pi-data",
      "kind": "node",
      "label": "PI robot data"
    },
    "train": {
      "id": "mechanism-train",
      "kind": "node",
      "label": "Robot training"
    },
    "policy": {
      "id": "mechanism-policy",
      "kind": "node",
      "label": "Generalist policy"
    },
    "pretrain": {
      "id": "mechanism-pretrain",
      "kind": "edge",
      "label": "Pretrain VLM foundation"
    },
    "initialize": {
      "id": "mechanism-initialize",
      "kind": "edge",
      "label": "Initialize robot training"
    },
    "openx-input": {
      "id": "mechanism-openx-input",
      "kind": "edge",
      "label": "Use open robot data"
    },
    "pi-input": {
      "id": "mechanism-pi-input",
      "kind": "edge",
      "label": "Use PI robot data"
    },
    "trained-policy": {
      "id": "mechanism-trained-policy",
      "kind": "edge",
      "label": "Keep the trained policy"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "training",
      "title": "Foundation initialization and robot training",
      "description": "The diagram gives the training recipe. It does not give a layer-exact optimizer trace.",
      "initial": {
        "stage": "Training recipe",
        "artifact": "Internet data and robot datasets available"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Pretrain the visual-language foundation",
          "description": "Internet-scale vision-language pretraining supplies the foundation weights.",
          "duration": 3000,
          "active": [
            "foundation"
          ],
          "paths": [
            "pretrain"
          ],
          "state": {
            "stage": "Foundation available",
            "artifact": "Pretrained VLM weights"
          }
        },
        {
          "title": "Initialize robot training from the foundation",
          "description": "Robot training starts from the pretrained foundation weights.",
          "duration": 3000,
          "active": [
            "train"
          ],
          "paths": [
            "initialize"
          ],
          "state": {
            "stage": "Initialize policy",
            "artifact": "Foundation weights before robot training"
          }
        },
        {
          "title": "Train with the two robot-data sources",
          "description": "Robot training uses Open X-Embodiment and PI robot data.",
          "duration": 3000,
          "active": [
            "train"
          ],
          "paths": [
            "openx-input",
            "pi-input"
          ],
          "state": {
            "stage": "Robot training",
            "artifact": "Open X-Embodiment and PI robot data"
          }
        },
        {
          "title": "Keep the trained generalist policy",
          "description": "The trained generalist policy is kept as the result.",
          "duration": 3000,
          "active": [
            "policy"
          ],
          "paths": [
            "trained-policy"
          ],
          "state": {
            "stage": "Training completed",
            "artifact": "Generalist π0 policy"
          }
        }
      ]
    }
  ]
}