{
  "version": 1,
  "language": "en",
  "title": "Attention weights mix a separate value branch",
  "description": "Q and K give scaled scores. A causal mask and softmax give attention weights, which mix the separate V branch.",
  "caption": "Q and K give scaled scores. Masking and softmax give attention weights from these scores. V supplies the values on a separate branch. The author made this explanatory diagram to show data dependencies, not a kernel launch schedule. The playback is an example. Animation time is not measured runtime.",
  "alt": "Flow diagram: Q and K give masked attention weights. A separate V branch supplies the values for their weighted sum.",
  "sources": [
    {
      "path": "content/blog/2026-06-09_life-of-a-token-transformer/post.md",
      "sha256": "aca6d72d9a2cb565f5e2e514e33cc16c3f73abb24785570c2dd1a2ea672af45d",
      "description": "Current article explanation, equations, assumptions and retained citations."
    },
    {
      "path": "research/dynamic-mechanisms-2026-10-21/before/content/blog/2026-06-09_life-of-a-token-transformer/illustrations/02-attention-value-routing-algorithm.svg",
      "sha256": "9da59fcf5bea47b07565957c9942fb99bf9aff7ab9e88afad3d217408058e8eb",
      "description": "Frozen prior illustration; exact pre-migration bytes retained."
    },
    {
      "path": "content/blog/2026-06-09_life-of-a-token-transformer/illustrations/source-notes.json",
      "description": "Existing article illustration provenance, retained unchanged."
    }
  ],
  "limitations": [
    "The playback is an example. Playback duration is not measured runtime."
  ],
  "targets": {
    "x": {
      "id": "mechanism-x",
      "kind": "node",
      "label": "Hidden X"
    },
    "q": {
      "id": "mechanism-q",
      "kind": "node",
      "label": "Q"
    },
    "k": {
      "id": "mechanism-k",
      "kind": "node",
      "label": "K"
    },
    "v": {
      "id": "mechanism-v",
      "kind": "node",
      "label": "V"
    },
    "scores": {
      "id": "mechanism-scores",
      "kind": "node",
      "label": "Scores"
    },
    "softmax": {
      "id": "mechanism-softmax",
      "kind": "node",
      "label": "Mask + softmax"
    },
    "mix": {
      "id": "mechanism-mix",
      "kind": "node",
      "label": "Weighted sum"
    },
    "xq": {
      "id": "mechanism-xq",
      "kind": "edge",
      "label": "project queries"
    },
    "xk": {
      "id": "mechanism-xk",
      "kind": "edge",
      "label": "project keys"
    },
    "xv": {
      "id": "mechanism-xv",
      "kind": "edge",
      "label": "project values"
    },
    "qscore": {
      "id": "mechanism-qscore",
      "kind": "edge",
      "label": "score queries"
    },
    "kscore": {
      "id": "mechanism-kscore",
      "kind": "edge",
      "label": "score keys"
    },
    "mask": {
      "id": "mechanism-mask",
      "kind": "edge",
      "label": "apply causal mask"
    },
    "weights": {
      "id": "mechanism-weights",
      "kind": "edge",
      "label": "apply weights"
    },
    "values": {
      "id": "mechanism-values",
      "kind": "edge",
      "label": "supply values"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "causal-attention",
      "title": "Attention weights mix values",
      "description": "Attention weights mix values.",
      "initial": {
        "stage": "Prepared",
        "artifact": "Inputs kept"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Project Q, K and V",
          "description": "The model projects the same hidden states into Q, K and V.",
          "duration": 3000,
          "active": [
            "q",
            "k",
            "v"
          ],
          "paths": [
            "xq",
            "xk",
            "xv"
          ],
          "state": {
            "stage": "Project Q, K and V",
            "artifact": "Three projections of the same hidden states"
          }
        },
        {
          "title": "Calculate scaled scores",
          "description": "The model calculates scaled query-key scores.",
          "duration": 3000,
          "active": [
            "scores"
          ],
          "paths": [
            "qscore",
            "kscore"
          ],
          "state": {
            "stage": "Calculate scaled scores",
            "artifact": "Query-key scores before causal filtering"
          }
        },
        {
          "title": "Apply mask and normalize",
          "description": "The causal mask prevents attention to future positions. Softmax gives attention weights from the masked scores.",
          "duration": 3000,
          "active": [
            "softmax"
          ],
          "paths": [
            "mask"
          ],
          "state": {
            "stage": "Apply mask and normalize",
            "artifact": "Permitted attention weights"
          }
        },
        {
          "title": "Mix the value branch",
          "description": "Attention weights mix the values from the separate V branch.",
          "duration": 3000,
          "active": [
            "mix"
          ],
          "paths": [
            "weights",
            "values"
          ],
          "state": {
            "stage": "Mix the value branch",
            "artifact": "Attention output"
          }
        }
      ]
    }
  ]
}