/**
* Verify the shipped extension against the live Hyper API and models.dev.
*
* npm run verify
*
* Three claims are checked:
*
* 1. mapping the catalog row -> Prime Agent model config derivation, pure code
* 2. catalog catalog.json still agrees with /v1/models and models.dev's hyper entry
* 3. wire streaming, usage, prompt-cache accounting, tool calls, reasoning output,
* image handling, and the credit balance the /hyper command reads
*
* Roughly twenty requests. Exits non-zero on any failure.
*/
import { readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { type ModelFacts, fetchDevModels, parseFactsResponse, toProviderModel } from "./catalog.js";
import { API_BASE_URL, ENV_VAR, PROVIDER, fetchBalance, resolveKey } from "./hyper.js";
import { authorization, getJson } from "./http.js";
const CATALOG_PATH = join(dirname(fileURLToPath(import.meta.url)), "catalog.json");
const failures: string[] = [];
function expect(condition: boolean, message: string): void {
if (!condition) failures.push(message);
}
function readApiKey(): string {
const key = resolveKey();
if (!key) throw new Error(`no API key: run /login ${PROVIDER} or set ${ENV_VAR}`);
return key;
}
const apiKey = readApiKey();
const catalog = JSON.parse(readFileSync(CATALOG_PATH, "utf-8")) as { models: ModelFacts[] };
const byId = new Map(catalog.models.map((model) => [model.id, model]));
const headers = { ...authorization(apiKey), "Content-Type": "application/json" };
interface Completion {
status: number;
reasoning: string;
text: string;
toolCalls: number;
usage: Record<string, unknown>;
}
async function complete(model: string, _prompt: string, extra: Record<string, unknown> = {}): Promise<Completion> {
const response = await fetch(`${API_BASE_URL}/chat/completions`, {
method: "POST",
headers,
body: JSON.stringify({ model, max_tokens: 96, ...extra }),
});
const body = (await response.json()) as {
choices?: Array<{ message?: { reasoning_content?: unknown; content?: unknown; tool_calls?: unknown[] } }>;
usage?: Record<string, unknown>;
};
const message = body.choices?.[0]?.message;
return {
status: response.status,
reasoning: typeof message?.reasoning_content === "string" ? message.reasoning_content : "",
text: typeof message?.content === "string" ? message.content : "",
toolCalls: message?.tool_calls?.length ?? 0,
usage: body.usage ?? {},
};
}
const SAMPLE = "qwen3.8-flash";
const OTHER = "deepseek-v4-flash";
const TRIVIAL_PROMPT = "Reply with the single word: ok.";
const REASONING_PROMPT = "Compute 47*89 and 1234*5678. Show your work.";
// 1. mapping --------------------------------------------------------------------------
function checkMapping(): void {
const reasoning: ModelFacts = {
id: "m",
name: "m",
reasoning: true,
vision: false,
contextWindow: 1000,
maxTokens: 100,
levels: ["high", "max"],
cost: { input: 1, output: 2, cacheRead: 0.1, cacheWrite: 0 },
};
const withLevels = toProviderModel(reasoning);
expect(withLevels.thinkingLevelMap?.high === "high", "an advertised level must map to itself");
expect(withLevels.thinkingLevelMap?.low === null, "an unadvertised level must be hidden");
expect(withLevels.thinkingLevelMap?.off === null, "an off switch must not be invented");
const withOff = toProviderModel({ ...reasoning, levels: ["none", "medium"] });
expect(withOff.thinkingLevelMap?.off === "none", "an advertised none must become the off level");
expect(withOff.thinkingLevelMap?.medium === "medium", "an advertised level must map to itself");
expect(withOff.thinkingLevelMap?.max === null, "an unadvertised level must be hidden");
const nameless = toProviderModel({ ...reasoning, levels: [] });
expect(nameless.thinkingLevelMap?.max === "max", "a model that reasons without levels needs one on state");
expect(nameless.thinkingLevelMap?.low === null, "a model that reasons without levels offers nothing else");
const plain = toProviderModel({ ...reasoning, reasoning: false, levels: [] });
expect(plain.thinkingLevelMap === undefined, "a non-reasoning model must have no thinking controls");
expect(plain.compat?.supportsReasoningEffort === false, "a non-reasoning model must not send reasoning_effort");
expect(plain.compat?.maxTokensField === "max_tokens", "Hyper takes max_tokens");
expect(plain.compat?.supportsStore === false, "Hyper has no response store");
}
// 2. catalog --------------------------------------------------------------------------
const close = (a: number, b: number): boolean => Math.abs(a - b) < 1e-4;
async function checkCatalog(): Promise<void> {
const live = parseFactsResponse(await getJson(`${API_BASE_URL}/models`), await fetchDevModels());
const liveIds = new Set(live.map((model) => model.id));
for (const model of live) {
const frozen = byId.get(model.id);
expect(frozen !== undefined, `${model.id} is missing from catalog.json (run npm run regen)`);
if (!frozen) continue;
expect(frozen.name === model.name, `${model.id} name changed`);
expect(close(frozen.cost.input, model.cost.input), `${model.id} input price changed`);
expect(close(frozen.cost.output, model.cost.output), `${model.id} output price changed`);
expect(close(frozen.cost.cacheRead, model.cost.cacheRead), `${model.id} cache read price changed`);
expect(frozen.contextWindow === model.contextWindow, `${model.id} context window changed`);
expect(frozen.maxTokens === model.maxTokens, `${model.id} output limit changed`);
expect(frozen.vision === model.vision, `${model.id} vision support changed`);
expect(frozen.reasoning === model.reasoning, `${model.id} reasoning support changed`);
expect(
frozen.levels.join(",") === model.levels.join(","),
`${model.id} effort levels changed (${frozen.levels.join(",")} -> ${model.levels.join(",")})`,
);
}
for (const model of catalog.models) {
expect(liveIds.has(model.id), `${model.id} is in catalog.json but not in /v1/models (run npm run regen)`);
}
}
// 3. wire -----------------------------------------------------------------------------
async function streamed(body: Record<string, unknown>): Promise<{ status: number; events: string[] }> {
const response = await fetch(`${API_BASE_URL}/chat/completions`, {
method: "POST",
headers,
body: JSON.stringify({ ...body, stream: true, stream_options: { include_usage: true } }),
});
const events: string[] = [];
for (const line of (await response.text()).split("\n")) {
if (line.startsWith("data: ")) events.push(line.slice(6));
}
return { status: response.status, events };
}
async function checkWire(): Promise<void> {
const stream = await streamed({
model: SAMPLE,
messages: [{ role: "user", content: TRIVIAL_PROMPT }],
max_tokens: 32,
});
expect(stream.status === 200, `streaming ${SAMPLE} failed (${stream.status})`);
const parsed = stream.events.filter((event) => event !== "[DONE]").map((event) => JSON.parse(event));
expect(parsed.length > 1, "streaming must produce more than one chunk");
expect(parsed.some((chunk: { choices?: Array<{ delta?: { content?: unknown } }> }) => typeof chunk.choices?.[0]?.delta?.content === "string"), "streaming must emit text deltas");
const last = parsed.at(-1) as { usage?: Record<string, unknown> };
expect(last.usage !== undefined, "the final chunk must carry usage when include_usage is set");
expect(typeof (last.usage?.cost as { usd?: unknown } | undefined)?.usd === "number", "usage must report cost");
// A shared prefix must be billed as a cache read on the second call, which is what
// prompt_tokens_details.cached_tokens feeds into Prime Agent's cacheRead usage.
const prefix = `${"The quick brown fox jumps over the lazy dog. ".repeat(40)}\n`.repeat(8);
const messages = [{ role: "system", content: prefix }, { role: "user", content: TRIVIAL_PROMPT }];
await complete(SAMPLE, "", { messages, max_tokens: 16 });
const warm = await complete(SAMPLE, "", { messages, max_tokens: 16 });
const details = warm.usage.prompt_tokens_details as { cached_tokens?: number } | undefined;
expect((details?.cached_tokens ?? 0) > 0, "a repeated prefix must report cached_tokens");
const tools = [
{
type: "function",
function: {
name: "get_weather",
description: "Get the weather",
parameters: { type: "object", properties: { city: { type: "string" } }, required: ["city"] },
strict: false,
},
},
];
const toolCall = await complete(SAMPLE, "", {
messages: [{ role: "user", content: "What is the weather in Tokyo? Use the tool." }],
tools,
max_tokens: 64,
});
expect(toolCall.status === 200, `a tool request failed (${toolCall.status})`);
expect(toolCall.toolCalls > 0, "a tool request must return tool_calls");
const thinking = await complete(OTHER, "", {
messages: [{ role: "user", content: REASONING_PROMPT }],
reasoning_effort: "high",
max_tokens: 96,
});
expect(thinking.status === 200, `a reasoning request failed (${thinking.status})`);
expect(thinking.reasoning !== "", "a reasoning model at an effort level must return reasoning_content");
const image = `data:image/png;base64,${"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFAAH/q842iQAAAABJRU5ErkJggg=="}`;
const content = [
{ type: "text", text: "What color is this pixel?" },
{ type: "image_url", image_url: { url: image } },
];
const visionModel = catalog.models.find((model) => model.vision);
if (visionModel) {
const accepted = await complete(visionModel.id, "", {
messages: [{ role: "user", content }],
max_tokens: 32,
});
expect(accepted.status === 200, `${visionModel.id} is listed as vision capable but rejected an image`);
}
// Hyper enforces attachment support per model, and only some non-vision models answer
// with the documented 400 (deepseek-v4-flash answers 200 and ignores the image), so the
// check names one that enforces it.
const enforcing = "gemma-4-26b-a4b-it";
const rejected = await complete(enforcing, "", { messages: [{ role: "user", content }], max_tokens: 32 });
expect(rejected.status === 400, `${enforcing} did not reject an image (${rejected.status})`);
const balance = await fetchBalance(apiKey);
expect(typeof balance === "number", "GET /v1/credits must return a numeric balance for the /hyper command");
}
async function main(): Promise<void> {
checkMapping();
process.stdout.write("mapping checked\n");
await checkCatalog();
process.stdout.write("catalog checked against /v1/models and models.dev\n");
await checkWire();
process.stdout.write("wire behaviour checked\n");
if (failures.length > 0) {
for (const failure of failures) process.stdout.write(`FAIL ${failure}\n`);
process.exitCode = 1;
return;
}
process.stdout.write(`ok: ${catalog.models.length} models verified against ${API_BASE_URL}\n`);
}
await main();