{
  "version": 1,
  "language": "en",
  "title": "Residual paths keep each sublayer input",
  "description": "A representative pre-normalized decoder applies attention and an MLP. Each residual path keeps its sublayer input unchanged.",
  "caption": "Two pre-normalized residual sublayers use attention and a gated MLP. Dashed residual edges keep each sublayer input unchanged. This is a representative decoder, not a universal architecture. The author made this explanatory diagram. The playback is an example. Animation time is not measured runtime.",
  "alt": "Flow diagram: a pre-normalized decoder applies attention and an MLP, each with its unchanged residual input.",
  "sources": [
    {
      "path": "content/blog/2026-06-09_life-of-a-token-transformer/post.md",
      "sha256": "aca6d72d9a2cb565f5e2e514e33cc16c3f73abb24785570c2dd1a2ea672af45d",
      "description": "Current article explanation, equations, assumptions and retained citations."
    },
    {
      "path": "research/dynamic-mechanisms-2026-10-21/before/content/blog/2026-06-09_life-of-a-token-transformer/illustrations/01-decoder-residual-block-algorithm.svg",
      "sha256": "b7165cdb7469f6f1f355ccbf8d49cd8de6b93bdcf37822c92fffcb8ab557221d",
      "description": "Frozen prior illustration; exact pre-migration bytes retained."
    },
    {
      "path": "content/blog/2026-06-09_life-of-a-token-transformer/illustrations/source-notes.json",
      "description": "Existing article illustration provenance, retained unchanged."
    }
  ],
  "limitations": [
    "The playback is an example. Playback duration is not measured runtime."
  ],
  "targets": {
    "input": {
      "id": "mechanism-input",
      "kind": "node",
      "label": "Input state"
    },
    "norm1": {
      "id": "mechanism-norm1",
      "kind": "node",
      "label": "Normalize"
    },
    "attention": {
      "id": "mechanism-attention",
      "kind": "node",
      "label": "Attention"
    },
    "add1": {
      "id": "mechanism-add1",
      "kind": "node",
      "label": "Residual add"
    },
    "norm2": {
      "id": "mechanism-norm2",
      "kind": "node",
      "label": "Normalize"
    },
    "mlp": {
      "id": "mechanism-mlp",
      "kind": "node",
      "label": "Gated MLP"
    },
    "add2": {
      "id": "mechanism-add2",
      "kind": "node",
      "label": "Residual add"
    },
    "input-norm": {
      "id": "mechanism-input-norm",
      "kind": "edge",
      "label": "normalize input"
    },
    "norm-attention": {
      "id": "mechanism-norm-attention",
      "kind": "edge",
      "label": "calculate attention"
    },
    "attention-add": {
      "id": "mechanism-attention-add",
      "kind": "edge",
      "label": "add attention"
    },
    "skip-attention": {
      "id": "mechanism-skip-attention",
      "kind": "edge",
      "label": "keep input"
    },
    "next-norm": {
      "id": "mechanism-next-norm",
      "kind": "edge",
      "label": "normalize state"
    },
    "norm-mlp": {
      "id": "mechanism-norm-mlp",
      "kind": "edge",
      "label": "apply MLP"
    },
    "mlp-add": {
      "id": "mechanism-mlp-add",
      "kind": "edge",
      "label": "add MLP"
    },
    "skip-mlp": {
      "id": "mechanism-skip-mlp",
      "kind": "edge",
      "label": "keep state"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "residual-block",
      "title": "Two residual sublayers",
      "description": "Two residual sublayers",
      "initial": {
        "stage": "Prepared",
        "artifact": "Inputs kept"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Normalize the input",
          "description": "The block normalizes the input.",
          "duration": 3000,
          "active": [
            "input",
            "norm1"
          ],
          "paths": [
            "input-norm"
          ],
          "state": {
            "stage": "Normalize the input",
            "artifact": "Per-token normalized input"
          }
        },
        {
          "title": "Mix permitted positions",
          "description": "Attention mixes information from permitted positions.",
          "duration": 3000,
          "active": [
            "attention"
          ],
          "paths": [
            "norm-attention"
          ],
          "state": {
            "stage": "Mix permitted positions",
            "artifact": "Attention branch output"
          }
        },
        {
          "title": "Add the unchanged input",
          "description": "The residual path adds the unchanged input to the attention output.",
          "duration": 3000,
          "active": [
            "add1"
          ],
          "paths": [
            "attention-add",
            "skip-attention"
          ],
          "state": {
            "stage": "Add the unchanged input",
            "artifact": "State after attention residual"
          }
        },
        {
          "title": "Normalize that state",
          "description": "The block normalizes the state after the attention residual.",
          "duration": 3000,
          "active": [
            "norm2"
          ],
          "paths": [
            "next-norm"
          ],
          "state": {
            "stage": "Normalize that state",
            "artifact": "Second sublayer input kept for skip"
          }
        },
        {
          "title": "Apply the gated MLP",
          "description": "The gated MLP transforms each position's vector.",
          "duration": 3000,
          "active": [
            "mlp"
          ],
          "paths": [
            "norm-mlp"
          ],
          "state": {
            "stage": "Apply the gated MLP",
            "artifact": "Per-position MLP output"
          }
        },
        {
          "title": "Add the sublayer input",
          "description": "The residual path adds that sublayer's unchanged input to the MLP output.",
          "duration": 3000,
          "active": [
            "add2"
          ],
          "paths": [
            "mlp-add",
            "skip-mlp"
          ],
          "state": {
            "stage": "Add the sublayer input",
            "artifact": "Next hidden state"
          }
        }
      ]
    }
  ]
}