One token at a time
Given the text so far, a model outputs a score (logit) for every token in its vocabulary. Softmax turns scores into probabilities, then one token is sampled, appended, and the whole thing repeats. Temperature sharpens or flattens the distribution; top-p cuts off the unlikely tail. That's why the same prompt can give different answers.
From scores to a choice
function softmax(logits, temperature = 1) {
if (temperature === 0) {
const best = logits.indexOf(Math.max(...logits));
return logits.map((_, i) => (i === best ? 1 : 0)); // greedy
}
const scaled = logits.map((l) => l / temperature);
const max = Math.max(...scaled); // subtract the max so exp never overflows
const exps = scaled.map((s) => Math.exp(s - max));
const sum = exps.reduce((a, b) => a + b, 0);
return exps.map((e) => e / sum);
}
function topP(probs, p) {
const order = probs.map((prob, i) => [prob, i]).sort((a, b) => b[0] - a[0]);
const keep = new Set();
let total = 0;
for (const [prob, i] of order) {
keep.add(i);
total += prob;
if (total >= p) break; // the smallest set covering p of the probability
}
const kept = probs.map((prob, i) => (keep.has(i) ? prob : 0));
const sum = kept.reduce((a, b) => a + b, 0);
return kept.map((x) => x / sum);
}
function sample(probs, random = Math.random) {
let x = random();
for (let i = 0; i < probs.length; i++) {
x -= probs[i];
if (x < 0) return i;
}
return probs.length - 1;
}