{
  "version": 1,
  "language": "en",
  "title": "HumanEgo: predict actions from hand–object relations",
  "description": "A shared encoder and flow-matching policy use RGB observations and entity-relative hand tokens. The policy generates bimanual actions. Different training-only heads predict object motion, 2D traces and future tokens.",
  "caption": "The author made this diagram from the source figure. Entity tokens contain hand poses in entity frames. The action generator uses RGB and relational context. Dashed auxiliary heads supply training supervision only. The diagram does not show source photographs, prediction plots or task-success measurements.",
  "alt": "The policy flow uses RGB and hand–object tokens to generate bimanual actions, with different prediction heads for training only.",
  "sources": [
    "https://arxiv.org/abs/2605.24934",
    "https://humanego-ai.github.io/"
  ],
  "limitations": [
    "The playback is an example. Playback time is not measured runtime."
  ],
  "targets": {
    "rgb": {
      "id": "mechanism-rgb",
      "kind": "node",
      "label": "RGB observation"
    },
    "visual": {
      "id": "mechanism-visual",
      "kind": "node",
      "label": "Visual encoder"
    },
    "shared": {
      "id": "mechanism-shared",
      "kind": "node",
      "label": "Shared context"
    },
    "tracked": {
      "id": "mechanism-tracked",
      "kind": "node",
      "label": "Tracked entities"
    },
    "ict": {
      "id": "mechanism-ict",
      "kind": "node",
      "label": "Entity tokens"
    },
    "policy": {
      "id": "mechanism-policy",
      "kind": "node",
      "label": "Flow-matching policy"
    },
    "object-head": {
      "id": "mechanism-object-head",
      "kind": "node",
      "label": "Object motion"
    },
    "trace-head": {
      "id": "mechanism-trace-head",
      "kind": "node",
      "label": "2D traces"
    },
    "ict-head": {
      "id": "mechanism-ict-head",
      "kind": "node",
      "label": "Future ICT"
    },
    "encode-rgb": {
      "id": "mechanism-encode-rgb",
      "kind": "edge",
      "label": "Encode RGB"
    },
    "visual-context": {
      "id": "mechanism-visual-context",
      "kind": "edge",
      "label": "Supply visual context"
    },
    "form-tokens": {
      "id": "mechanism-form-tokens",
      "kind": "edge",
      "label": "Make tokens"
    },
    "token-context": {
      "id": "mechanism-token-context",
      "kind": "edge",
      "label": "Supply tokens"
    },
    "generate-actions": {
      "id": "mechanism-generate-actions",
      "kind": "edge",
      "label": "Generate actions"
    },
    "training-features": {
      "id": "mechanism-training-features",
      "kind": "edge",
      "label": "Supply training features"
    },
    "future-object": {
      "id": "mechanism-future-object",
      "kind": "edge",
      "label": "Predict object motion"
    },
    "future-trace": {
      "id": "mechanism-future-trace",
      "kind": "edge",
      "label": "Predict traces"
    },
    "future-ict": {
      "id": "mechanism-future-ict",
      "kind": "edge",
      "label": "Predict tokens"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "policy-inference",
      "title": "Policy inference",
      "description": "The example shows policy inference.",
      "initial": {
        "stage": "Start",
        "artifact": "Available inputs"
      },
      "unselected": [
        "training-features",
        "future-object",
        "future-trace",
        "future-ict",
        "object-head",
        "trace-head",
        "ict-head"
      ],
      "steps": [
        {
          "title": "Encode the RGB observation",
          "description": "Encode the RGB observation.",
          "duration": 3000,
          "active": [
            "visual"
          ],
          "paths": [
            "encode-rgb"
          ],
          "state": {
            "stage": "Visual context",
            "artifact": "Encoded appearance"
          }
        },
        {
          "title": "Make interaction-centric tokens",
          "description": "Make interaction-centric tokens.",
          "duration": 3000,
          "active": [
            "ict"
          ],
          "paths": [
            "form-tokens"
          ],
          "state": {
            "stage": "Interaction context",
            "artifact": "Entity identity, pose, grasp and relative hand poses"
          }
        },
        {
          "title": "Make one representation",
          "description": "Make one representation from the two contexts.",
          "duration": 3000,
          "active": [
            "shared"
          ],
          "paths": [
            "visual-context",
            "token-context"
          ],
          "state": {
            "stage": "Shared representation",
            "artifact": "Appearance plus interaction geometry"
          }
        },
        {
          "title": "Generate the action horizon",
          "description": "Generate the action horizon.",
          "duration": 3000,
          "active": [
            "policy"
          ],
          "paths": [
            "generate-actions"
          ],
          "state": {
            "stage": "Policy output",
            "artifact": "Bimanual poses and grasp commands"
          }
        }
      ]
    },
    {
      "id": "auxiliary-training",
      "title": "Auxiliary training",
      "description": "The example shows auxiliary training.",
      "initial": {
        "stage": "Start",
        "artifact": "Available inputs"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Encode the RGB observation",
          "description": "Encode the RGB observation.",
          "duration": 3000,
          "active": [
            "visual"
          ],
          "paths": [
            "encode-rgb"
          ],
          "state": {
            "stage": "Visual context",
            "artifact": "Encoded appearance"
          }
        },
        {
          "title": "Make interaction-centric tokens",
          "description": "Make interaction-centric tokens.",
          "duration": 3000,
          "active": [
            "ict"
          ],
          "paths": [
            "form-tokens"
          ],
          "state": {
            "stage": "Interaction context",
            "artifact": "Entity identity, pose, grasp and relative hand poses"
          }
        },
        {
          "title": "Make one representation",
          "description": "Make one representation from the two contexts.",
          "duration": 3000,
          "active": [
            "shared"
          ],
          "paths": [
            "visual-context",
            "token-context"
          ],
          "state": {
            "stage": "Shared representation",
            "artifact": "Appearance plus interaction geometry"
          }
        },
        {
          "title": "Keep the action objective",
          "description": "Keep the action objective.",
          "duration": 3000,
          "active": [
            "policy"
          ],
          "paths": [
            "generate-actions"
          ],
          "state": {
            "stage": "Action supervision",
            "artifact": "Action-horizon training target"
          }
        },
        {
          "title": "Supply shared features",
          "description": "Supply shared features.",
          "duration": 3000,
          "active": [],
          "paths": [
            "training-features"
          ],
          "state": {
            "stage": "Auxiliary supervision",
            "artifact": "The shared representation supplies training-only prediction heads."
          }
        },
        {
          "title": "Predict future interaction outcomes",
          "description": "Predict future interaction outcomes.",
          "duration": 3000,
          "active": [
            "object-head",
            "trace-head",
            "ict-head"
          ],
          "paths": [
            "future-object",
            "future-trace",
            "future-ict"
          ],
          "state": {
            "stage": "Future targets",
            "artifact": "The targets are future object motion, 2D traces and ICT state. They do not add an execution command."
          }
        }
      ]
    }
  ]
}