{
  "version": 1,
  "language": "en",
  "title": "V-JEPA predicts held-out features",
  "description": "The context and target encoders operate in parallel. The predictor gives features for comparison with stop-gradient target features. The training update uses context-encoder parameters to update target-encoder weights with an exponential moving average (EMA). Target-position selection aligns the two feature sets.",
  "caption": "V-JEPA learns to predict held-out target embeddings from visible video context. The target encoder uses an exponential moving average (EMA), and its features are stop-gradient targets. Author-redrawn mechanism from Bardes et al., without token grids and implementation shapes. Source: Bardes et al. Playback time is illustrative and is not measured runtime.",
  "alt": "V-JEPA training flow: context predicts held-out features, and an EMA encoder gives stop-gradient targets.",
  "sources": [
    "https://arxiv.org/abs/2404.08471"
  ],
  "limitations": [
    "This playback is an example. Playback duration is not measured runtime.",
    "The diagram does not include detailed optimizer and reverse-gradient paths for context/predictor parameters.",
    "The target encoder receives EMA weights, not the feature-loss gradient."
  ],
  "targets": {
    "context-input": {
      "id": "mechanism-context-input",
      "kind": "node",
      "label": "Video context"
    },
    "context-encoder": {
      "id": "mechanism-context-encoder",
      "kind": "node",
      "label": "Context encoder"
    },
    "predictor": {
      "id": "mechanism-predictor",
      "kind": "node",
      "label": "Predictor"
    },
    "target-input": {
      "id": "mechanism-target-input",
      "kind": "node",
      "label": "Full clip"
    },
    "target-encoder": {
      "id": "mechanism-target-encoder",
      "kind": "node",
      "label": "Target encoder"
    },
    "target-select": {
      "id": "mechanism-target-select",
      "kind": "node",
      "label": "Target selection"
    },
    "feature-loss": {
      "id": "mechanism-feature-loss",
      "kind": "node",
      "label": "Feature loss"
    },
    "context-encode": {
      "id": "mechanism-context-encode",
      "kind": "edge",
      "label": "encode context"
    },
    "context-predict": {
      "id": "mechanism-context-predict",
      "kind": "edge",
      "label": "predict features"
    },
    "target-encode": {
      "id": "mechanism-target-encode",
      "kind": "edge",
      "label": "encode targets"
    },
    "target-select-edge": {
      "id": "mechanism-target-select-edge",
      "kind": "edge",
      "label": "select held-out targets"
    },
    "prediction-loss": {
      "id": "mechanism-prediction-loss",
      "kind": "edge",
      "label": "compare predictions"
    },
    "target-loss": {
      "id": "mechanism-target-loss",
      "kind": "edge",
      "label": "supply targets"
    },
    "ema-update": {
      "id": "mechanism-ema-update",
      "kind": "edge",
      "label": "update by EMA"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "feature-learning",
      "title": "Parallel feature learning",
      "description": "This trace shows parallel context/target encoding and an EMA target update.",
      "initial": {
        "stage": "Ready",
        "artifact": "Inputs available"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Video inputs",
          "description": "The two branches receive context and the full clip from the same video.",
          "duration": 3000,
          "active": [
            "context-input",
            "target-input"
          ],
          "paths": [],
          "state": {
            "stage": "Video inputs",
            "artifact": "Two views of the same video"
          }
        },
        {
          "title": "Parallel encoding",
          "description": "The context and target encoders encode the two branches in parallel.",
          "duration": 3000,
          "active": [
            "context-encoder",
            "target-encoder"
          ],
          "paths": [
            "context-encode",
            "target-encode"
          ],
          "state": {
            "stage": "Parallel encoding",
            "artifact": "Context and target embeddings"
          }
        },
        {
          "title": "Prediction and target selection",
          "description": "The predictor predicts features, and target-position selection aligns the held-out positions.",
          "duration": 3000,
          "active": [
            "predictor",
            "target-select"
          ],
          "paths": [
            "context-predict",
            "target-select-edge"
          ],
          "state": {
            "stage": "Prediction and target selection",
            "artifact": "Aligned held-out positions"
          }
        },
        {
          "title": "Feature comparison",
          "description": "The feature loss compares predictions with stop-gradient target features.",
          "duration": 3000,
          "active": [
            "feature-loss"
          ],
          "paths": [
            "prediction-loss",
            "target-loss"
          ],
          "state": {
            "stage": "Feature comparison",
            "artifact": "Stop-gradient feature target"
          }
        },
        {
          "title": "Target-weight update with EMA",
          "description": "The training process uses context-encoder parameters to update target weights with EMA. The target encoder does not receive the feature-loss gradient.",
          "duration": 3000,
          "active": [
            "target-encoder"
          ],
          "paths": [
            "ema-update"
          ],
          "state": {
            "stage": "Target-weight update with EMA",
            "artifact": "EMA weights. No target-loss gradient"
          }
        }
      ]
    }
  ]
}