{
  "version": 1,
  "language": "en",
  "title": "VIA: target selection and local execution",
  "description": "RGB-D observations update a visual interface. An agent reads screenshots and uses MCP tools to set gripper waypoints. A local backend runs the waypoints. The observation-only scenario executes no waypoint.",
  "caption": "The author made this interface mechanism diagram. Screenshots and MCP tools let the agent select a virtual gripper waypoint. The robot backend supplies local control. The example separates observation from execution. It does not include source screenshots, measured task results or direct joint control.",
  "alt": "The visual agent reads a robot interface, sends gripper-waypoint requests with MCP tools and reads local execution results.",
  "sources": [
    "https://arxiv.org/html/2607.11119v1"
  ],
  "limitations": [
    "The execution is an example. Playback duration is not measured runtime."
  ],
  "targets": {
    "agent": {
      "id": "mechanism-agent",
      "kind": "node",
      "label": "Visual agent"
    },
    "interface": {
      "id": "mechanism-interface",
      "kind": "node",
      "label": "Web interface"
    },
    "backend": {
      "id": "mechanism-backend",
      "kind": "node",
      "label": "Robot backend"
    },
    "screenshot": {
      "id": "mechanism-screenshot",
      "kind": "edge",
      "label": "Read the view"
    },
    "waypoint-request": {
      "id": "mechanism-waypoint-request",
      "kind": "edge",
      "label": "Send a waypoint request"
    },
    "rgbd-stream": {
      "id": "mechanism-rgbd-stream",
      "kind": "edge",
      "label": "Supply observations"
    },
    "execute-waypoint": {
      "id": "mechanism-execute-waypoint",
      "kind": "edge",
      "label": "Execute the waypoint"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Output",
    "executions": "Executed waypoints"
  },
  "scenarios": [
    {
      "id": "one-waypoint",
      "title": "One waypoint and fresh feedback",
      "description": "One waypoint and fresh feedback",
      "initial": {
        "stage": "Available",
        "artifact": "Available robot observations",
        "executions": "0"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Supply the robot observations",
          "description": "Supply the robot observations.",
          "duration": 3000,
          "active": [
            "interface"
          ],
          "paths": [
            "rgbd-stream"
          ],
          "state": {
            "stage": "Interface update",
            "artifact": "RGB-D and reconstructed views",
            "executions": "0"
          }
        },
        {
          "title": "Read the interface",
          "description": "Read the interface.",
          "duration": 3000,
          "active": [
            "agent"
          ],
          "paths": [
            "screenshot"
          ],
          "state": {
            "stage": "Visual reasoning",
            "artifact": "Screenshot from the engineered interface",
            "executions": "0"
          }
        },
        {
          "title": "Set a virtual gripper target",
          "description": "Set a virtual gripper target.",
          "duration": 3000,
          "active": [
            "interface"
          ],
          "paths": [
            "waypoint-request"
          ],
          "state": {
            "stage": "Tool interaction",
            "artifact": "The agent selects the position and rotation with MCP.",
            "executions": "0"
          }
        },
        {
          "title": "Execute with the local backend",
          "description": "Execute with the local backend.",
          "duration": 3000,
          "active": [
            "backend"
          ],
          "paths": [
            "execute-waypoint"
          ],
          "state": {
            "stage": "Physical execution",
            "artifact": "The controller runs the waypoint.",
            "executions": "1"
          }
        },
        {
          "title": "Update the views after motion",
          "description": "Update the views after motion.",
          "duration": 3000,
          "active": [
            "interface"
          ],
          "paths": [
            "rgbd-stream"
          ],
          "state": {
            "stage": "Feedback",
            "artifact": "Fresh robot observations after the waypoint",
            "executions": "1"
          }
        },
        {
          "title": "Examine the result before the next command",
          "description": "Examine the result before the next command.",
          "duration": 3000,
          "active": [
            "agent"
          ],
          "paths": [
            "screenshot"
          ],
          "state": {
            "stage": "Next decision",
            "artifact": "A new tool decision uses observed outcomes.",
            "executions": "1"
          }
        }
      ]
    },
    {
      "id": "observe-only",
      "title": "Read without execution",
      "description": "Read without execution.",
      "initial": {
        "stage": "Available",
        "artifact": "Available robot observations",
        "executions": "0"
      },
      "unselected": [
        "waypoint-request",
        "execute-waypoint"
      ],
      "steps": [
        {
          "title": "Supply the robot observations",
          "description": "Supply the robot observations.",
          "duration": 3000,
          "active": [
            "interface"
          ],
          "paths": [
            "rgbd-stream"
          ],
          "state": {
            "stage": "Interface update",
            "artifact": "RGB-D and reconstructed views",
            "executions": "0"
          }
        },
        {
          "title": "Read the interface",
          "description": "Read the interface.",
          "duration": 3000,
          "active": [
            "agent"
          ],
          "paths": [
            "screenshot"
          ],
          "state": {
            "stage": "Visual reasoning",
            "artifact": "Screenshot from the engineered interface",
            "executions": "0"
          }
        },
        {
          "title": "Complete the observation-only step",
          "description": "Complete the observation-only step.",
          "duration": 3000,
          "active": [
            "agent"
          ],
          "paths": [],
          "state": {
            "stage": "Completed observation-only step",
            "artifact": "The agent sent no execute_waypoint request.",
            "executions": "0"
          }
        }
      ]
    }
  ]
}