Models don't read letters
A large language model reads and writes tokens — common words and word pieces, each an id in a fixed vocabulary. They're learned by byte-pair encoding (BPE): start from single characters and repeatedly merge the most frequent neighbouring pair. Tokens explain prices (you pay per token), context limits, and why models are bad at counting letters.
Like texting shorthand: people who write "tomorrow" a lot start writing "tmrw" — frequent sequences earn their own symbol.
Training BPE
function mergePair(tokens, pair) {
const [a, b] = pair.split(" ");
const out = [];
for (let i = 0; i < tokens.length; i++) {
if (tokens[i] === a && tokens[i + 1] === b) {
out.push(a + b);
i++; // skip the second half
} else out.push(tokens[i]);
}
return out;
}
function trainBPE(text, numMerges) {
const counts = new Map();
for (const word of text.split(/\s+/).filter(Boolean)) counts.set(word, (counts.get(word) ?? 0) + 1);
let words = [...counts].map(([word, count]) => ({ tokens: [...word], count }));
const merges = [];
for (let m = 0; m < numMerges; m++) {
const pairs = new Map();
for (const { tokens, count } of words)
for (let i = 0; i < tokens.length - 1; i++) {
const pair = tokens[i] + " " + tokens[i + 1];
pairs.set(pair, (pairs.get(pair) ?? 0) + count);
}
let best = null;
for (const [pair, count] of pairs) if (!best || count > best[1]) best = [pair, count]; // most frequent, first wins ties
if (!best) break;
merges.push(best[0]);
words = words.map(({ tokens, count }) => ({ tokens: mergePair(tokens, best[0]), count }));
}
return merges;
}
const encode = (word, merges) => merges.reduce((tokens, pair) => mergePair(tokens, pair), [...word]);