{
  "version": 1,
  "language": "en",
  "title": "Representations before a new token",
  "description": "Examine representations from formatted text to logits. Then add the new sampled ID at the end of the sequence. Embed this ID for the next position.",
  "caption": "IDs, embeddings, contextual hidden states and logits are different representations. Sampling adds the next token to the end of the sequence and starts the next request step. Author-created explanatory diagram. The playback is an example. Animation time is not measured runtime.",
  "alt": "Representation flow: text becomes IDs, embeddings, hidden states and logits. Sampling adds a new token for the next position.",
  "sources": [
    {
      "path": "content/blog/2026-06-12_token-journey-across-llm-stack/post.md",
      "sha256": "66e83fe38f5b4077f193d245eae19f962b670be4bf39ee807dc5f8c02367c861",
      "description": "Current article explanation, equations, assumptions and retained citations."
    },
    {
      "path": "research/dynamic-mechanisms-2026-10-21/before/content/blog/2026-06-12_token-journey-across-llm-stack/illustrations/01-token-representations-algorithm.svg",
      "sha256": "01d9793a6708f2f45bb0b0b51b2c059a496076b760bc283515aee3d439630393",
      "description": "Frozen prior illustration; exact pre-migration bytes retained."
    },
    {
      "path": "content/blog/2026-06-12_token-journey-across-llm-stack/illustrations/source-notes.json",
      "description": "Existing article illustration provenance, retained unchanged."
    }
  ],
  "limitations": [
    "The execution is an example. Playback duration is not measured runtime."
  ],
  "targets": {
    "ids": {
      "id": "mechanism-ids",
      "kind": "node",
      "label": "Text"
    },
    "embed": {
      "id": "mechanism-embed",
      "kind": "node",
      "label": "IDs → vectors"
    },
    "transform": {
      "id": "mechanism-transform",
      "kind": "node",
      "label": "Transformer"
    },
    "final": {
      "id": "mechanism-final",
      "kind": "node",
      "label": "Prediction state"
    },
    "logits": {
      "id": "mechanism-logits",
      "kind": "node",
      "label": "Vocabulary logits"
    },
    "sample": {
      "id": "mechanism-sample",
      "kind": "node",
      "label": "New token ID"
    },
    "ids-embed": {
      "id": "mechanism-ids-embed",
      "kind": "edge",
      "label": "Tokenize and embed"
    },
    "embed-transform": {
      "id": "mechanism-embed-transform",
      "kind": "edge",
      "label": "Calculate contextual vectors"
    },
    "transform-final": {
      "id": "mechanism-transform-final",
      "kind": "edge",
      "label": "Select prediction state"
    },
    "final-logits": {
      "id": "mechanism-final-logits",
      "kind": "edge",
      "label": "Project to vocabulary"
    },
    "logits-sample": {
      "id": "mechanism-logits-sample",
      "kind": "edge",
      "label": "Sample a token ID"
    },
    "append": {
      "id": "mechanism-append",
      "kind": "edge",
      "label": "Add ID at the end for next position"
    }
  },
  "stateLabels": {
    "stage": "Stage",
    "artifact": "Artifact"
  },
  "scenarios": [
    {
      "id": "next-position",
      "title": "From current context to a new token",
      "description": "From current context to a new token",
      "initial": {
        "stage": "Prepared",
        "artifact": "Inputs kept"
      },
      "unselected": [],
      "steps": [
        {
          "title": "Format and tokenize",
          "description": "Formatting and tokenization give discrete IDs for the input positions.",
          "duration": 3000,
          "active": [
            "ids"
          ],
          "paths": [],
          "state": {
            "stage": "Format and tokenize",
            "artifact": "Discrete IDs for input positions"
          }
        },
        {
          "title": "Look up embeddings",
          "description": "Embedding lookup gives one vector for each input position.",
          "duration": 3000,
          "active": [
            "embed"
          ],
          "paths": [
            "ids-embed"
          ],
          "state": {
            "stage": "Look up embeddings",
            "artifact": "One vector for each input position"
          }
        },
        {
          "title": "Calculate contextual states",
          "description": "The Transformer calculates contextual hidden states.",
          "duration": 3000,
          "active": [
            "transform"
          ],
          "paths": [
            "embed-transform"
          ],
          "state": {
            "stage": "Calculate contextual states",
            "artifact": "Causal context affects hidden states."
          }
        },
        {
          "title": "Select the prediction position",
          "description": "The model selects the last input position for next-token prediction.",
          "duration": 3000,
          "active": [
            "final"
          ],
          "paths": [
            "transform-final"
          ],
          "state": {
            "stage": "Select the prediction position",
            "artifact": "The last input position predicts the next ID."
          }
        },
        {
          "title": "Project to vocabulary logits",
          "description": "The vocabulary projection gives one score for each vocabulary entry.",
          "duration": 3000,
          "active": [
            "logits"
          ],
          "paths": [
            "final-logits"
          ],
          "state": {
            "stage": "Project to vocabulary logits",
            "artifact": "One score for each vocabulary entry"
          }
        },
        {
          "title": "Sample a new ID",
          "description": "Sampling gives an ID for a new position.",
          "duration": 3000,
          "active": [
            "sample"
          ],
          "paths": [
            "logits-sample"
          ],
          "state": {
            "stage": "Sample a new ID",
            "artifact": "A new sequence position"
          }
        },
        {
          "title": "Add ID at the end and embed next input",
          "description": "The sampled ID is added at the end of the sequence. The next execution embeds this ID.",
          "duration": 3000,
          "active": [
            "embed"
          ],
          "paths": [
            "append"
          ],
          "state": {
            "stage": "Add ID at the end and embed next input",
            "artifact": "The new ID becomes input to the next execution."
          }
        }
      ]
    }
  ]
}