Models are chatty
Ask for JSON and you'll get "Sure! Here it is:" and a code fence — or a rating of "five". Programs need validated data. The pattern: pull the JSON out of the reply, check it against a schema, and if it fails, send the errors back and ask again — a few times at most.
Extract, validate, retry
function extractJson(text) {
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/);
const candidate = fenced ? fenced[1] : text.slice(text.search(/[[{]/), Math.max(text.lastIndexOf("}"), text.lastIndexOf("]")) + 1);
try {
return JSON.parse(candidate);
} catch {
throw new Error("No JSON found");
}
}
function validate(value, schema, path = "") {
const at = path || "value";
if (schema.enum) return schema.enum.includes(value) ? [] : [`${at}: expected one of ${schema.enum.join(", ")}`];
if (schema.type === "array") {
if (!Array.isArray(value)) return [`${at}: expected array`];
return value.flatMap((item, i) => validate(item, schema.items ?? {}, `${path}[${i}]`));
}
if (schema.type === "object") {
if (!value || typeof value !== "object" || Array.isArray(value)) return [`${at}: expected object`];
const join = (key) => (path ? `${path}.${key}` : key);
const missing = (schema.required ?? []).filter((k) => !(k in value)).map((k) => `${join(k)}: required`);
const nested = Object.entries(schema.properties ?? {}).flatMap(([k, s]) => (k in value ? validate(value[k], s, join(k)) : []));
return [...missing, ...nested];
}
if (schema.type && (typeof value !== schema.type || Number.isNaN(value))) return [`${at}: expected ${schema.type}`];
if (schema.min !== undefined && value < schema.min) return [`${at}: must be >= ${schema.min}`];
if (schema.max !== undefined && value > schema.max) return [`${at}: must be <= ${schema.max}`];
return [];
}
async function generateStructured(model, prompt, schema, { maxAttempts = 3 } = {}) {
const messages = [{ role: "user", content: prompt }];
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
const reply = await model(messages);
let errors;
try {
const value = extractJson(reply);
errors = validate(value, schema);
if (errors.length === 0) return { value, attempts: attempt };
} catch (e) {
errors = [e.message];
}
messages.push({ role: "assistant", content: reply }, { role: "user", content: `Your reply was invalid: ${errors.join("; ")}. Reply with JSON only.` }); // feed the errors back
}
throw new Error("Could not get valid output");
}const reviewSchema = {
type: "object",
required: ["dish", "rating", "mood"],
properties: {
dish: { type: "string" },
rating: { type: "number", min: 1, max: 5 },
mood: { enum: ["happy", "neutral", "sad"] },
tags: { type: "array", items: { type: "string" } },
},
};