import { type ChildProcess, execSync, spawn } from "child_process";
import { readFileSync } from "fs";
import { dirname, join } from "path";
import { Type } from "typebox";
import { fileURLToPath } from "url";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { getEnvApiKey } from "../src/env-api-keys.js";
import { getModel, getModels } from "../src/models.js";
import { complete, stream } from "../src/stream.js";
import type { Api, Context, ImageContent, Model, StreamOptions, Tool, ToolResultMessage } from "../src/types.js";
import { getKimiCodingTestModel } from "./kimi-test-model.js";
import { getZaiTestModel } from "./zai-test-model.js";
type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;
import { StringEnum } from "../src/utils/typebox-helpers.js";
import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.js";
import { hasBedrockCredentials } from "./bedrock-utils.js";
import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.js";
import { resolveApiKey } from "./oauth.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const oauthTokens = await Promise.all([
resolveApiKey("anthropic"),
resolveApiKey("github-copilot"),
resolveApiKey("openai-codex"),
]);
const [anthropicOAuthToken, githubCopilotToken, openaiCodexToken] = oauthTokens;
const primeInferenceApiKey = getEnvApiKey("prime-inference");
const calculatorSchema = Type.Object({
a: Type.Number({ description: "First number" }),
b: Type.Number({ description: "Second number" }),
operation: StringEnum(["add", "subtract", "multiply", "divide"], {
description: "The operation to perform. One of 'add', 'subtract', 'multiply', 'divide'.",
}),
});
const calculatorTool: Tool<typeof calculatorSchema> = {
name: "math_operation",
description: "Perform basic arithmetic operations",
parameters: calculatorSchema,
};
async function basicTextGeneration<TApi extends Api>(model: Model<TApi>, options?: StreamOptionsWithExtras) {
const context: Context = {
systemPrompt: "You are a helpful assistant. Be concise.",
messages: [{ role: "user", content: "Reply with exactly: 'Hello test successful'", timestamp: Date.now() }],
};
const response = await complete(model, context, options);
expect(response.role).toBe("assistant");
expect(response.content).toBeTruthy();
expect(response.usage.input + response.usage.cacheRead).toBeGreaterThan(0);
expect(response.usage.output).toBeGreaterThan(0);
expect(response.errorMessage).toBeFalsy();
expect(response.content.map((b) => (b.type === "text" ? b.text : "")).join("")).toContain("Hello test successful");
context.messages.push(response);
context.messages.push({ role: "user", content: "Now say 'Goodbye test successful'", timestamp: Date.now() });
const secondResponse = await complete(model, context, options);
expect(secondResponse.role).toBe("assistant");
expect(secondResponse.content).toBeTruthy();
expect(secondResponse.usage.input + secondResponse.usage.cacheRead).toBeGreaterThan(0);
expect(secondResponse.usage.output).toBeGreaterThan(0);
expect(secondResponse.errorMessage).toBeFalsy();
expect(secondResponse.content.map((b) => (b.type === "text" ? b.text : "")).join("")).toContain(
"Goodbye test successful",
);
}
async function handleToolCall<TApi extends Api>(model: Model<TApi>, options?: StreamOptionsWithExtras) {
const context: Context = {
systemPrompt: "You are a helpful assistant that uses tools when asked.",
messages: [
{
role: "user",
content: "Calculate 15 + 27 using the math_operation tool.",
timestamp: Date.now(),
},
],
tools: [calculatorTool],
};
const s = await stream(model, context, options);
let hasToolStart = false;
let hasToolDelta = false;
let hasToolEnd = false;
let accumulatedToolArgs = "";
let index = 0;
for await (const event of s) {
if (event.type === "toolcall_start") {
hasToolStart = true;
const toolCall = event.partial.content[event.contentIndex];
index = event.contentIndex;
expect(toolCall.type).toBe("toolCall");
if (toolCall.type === "toolCall") {
expect(toolCall.name).toBe("math_operation");
expect(toolCall.id).toBeTruthy();
}
}
if (event.type === "toolcall_delta") {
hasToolDelta = true;
const toolCall = event.partial.content[event.contentIndex];
expect(event.contentIndex).toBe(index);
expect(toolCall.type).toBe("toolCall");
if (toolCall.type === "toolCall") {
expect(toolCall.name).toBe("math_operation");
accumulatedToolArgs += event.delta;
expect(toolCall.arguments).toBeDefined();
expect(typeof toolCall.arguments).toBe("object");
expect(toolCall.arguments).not.toBeNull();
}
}
if (event.type === "toolcall_end") {
hasToolEnd = true;
const toolCall = event.partial.content[event.contentIndex];
expect(event.contentIndex).toBe(index);
expect(toolCall.type).toBe("toolCall");
if (toolCall.type === "toolCall") {
expect(toolCall.name).toBe("math_operation");
JSON.parse(accumulatedToolArgs);
expect(toolCall.arguments).not.toBeUndefined();
expect((toolCall.arguments as any).a).toBe(15);
expect((toolCall.arguments as any).b).toBe(27);
expect((toolCall.arguments as any).operation).oneOf(["add", "subtract", "multiply", "divide"]);
}
}
}
expect(hasToolStart).toBe(true);
expect(hasToolDelta).toBe(true);
expect(hasToolEnd).toBe(true);
const response = await s.result();
expect(response.stopReason).toBe("toolUse");
expect(response.content.some((b) => b.type === "toolCall")).toBeTruthy();
const toolCall = response.content.find((b) => b.type === "toolCall");
if (toolCall && toolCall.type === "toolCall") {
expect(toolCall.name).toBe("math_operation");
expect(toolCall.id).toBeTruthy();
} else {
throw new Error("No tool call found in response");
}
}
async function handleStreaming<TApi extends Api>(model: Model<TApi>, options?: StreamOptionsWithExtras) {
let textStarted = false;
let textChunks = "";
let textCompleted = false;
const context: Context = {
messages: [{ role: "user", content: "Count from 1 to 3", timestamp: Date.now() }],
systemPrompt: "You are a helpful assistant.",
};
const s = stream(model, context, options);
for await (const event of s) {
if (event.type === "text_start") {
textStarted = true;
} else if (event.type === "text_delta") {
textChunks += event.delta;
} else if (event.type === "text_end") {
textCompleted = true;
}
}
const response = await s.result();
expect(textStarted).toBe(true);
expect(textChunks.length).toBeGreaterThan(0);
expect(textCompleted).toBe(true);
expect(response.content.some((b) => b.type === "text")).toBeTruthy();
}
async function handleThinking<TApi extends Api>(model: Model<TApi>, options?: StreamOptionsWithExtras) {
let thinkingStarted = false;
let thinkingChunks = "";
let thinkingCompleted = false;
const context: Context = {
messages: [
{
role: "user",
content: `Think long and hard about ${(Math.random() * 255) | 0} + 27. Think step by step. Then output the result.`,
timestamp: Date.now(),
},
],
systemPrompt: "You are a helpful assistant.",
};
const s = stream(model, context, options);
for await (const event of s) {
if (event.type === "thinking_start") {
thinkingStarted = true;
} else if (event.type === "thinking_delta") {
thinkingChunks += event.delta;
} else if (event.type === "thinking_end") {
thinkingCompleted = true;
}
}
const response = await s.result();
expect(response.stopReason, `Error: ${response.errorMessage}`).toBe("stop");
expect(thinkingStarted).toBe(true);
expect(thinkingChunks.length).toBeGreaterThan(0);
expect(thinkingCompleted).toBe(true);
expect(response.content.some((b) => b.type === "thinking")).toBeTruthy();
}
async function handleImage<TApi extends Api>(model: Model<TApi>, options?: StreamOptionsWithExtras) {
if (!model.input.includes("image")) {
console.log(`Skipping image test - model ${model.id} doesn't support images`);
return;
}
const imagePath = join(__dirname, "data", "red-circle.png");
const imageBuffer = readFileSync(imagePath);
const base64Image = imageBuffer.toString("base64");
const imageContent: ImageContent = {
type: "image",
data: base64Image,
mimeType: "image/png",
};
const context: Context = {
messages: [
{
role: "user",
content: [
{
type: "text",
text: "What do you see in this image? Please describe the shape (circle, rectangle, square, triangle, ...) and color (red, blue, green, ...). You MUST reply in English.",
},
imageContent,
],
timestamp: Date.now(),
},
],
systemPrompt: "You are a helpful assistant.",
};
const response = await complete(model, context, options);
expect(response.content.length > 0).toBeTruthy();
const textContent = response.content.find((b) => b.type === "text");
if (textContent && textContent.type === "text") {
const lowerContent = textContent.text.toLowerCase();
expect(lowerContent).toContain("red");
expect(lowerContent).toContain("circle");
}
}
async function multiTurn<TApi extends Api>(model: Model<TApi>, options?: StreamOptionsWithExtras) {
const context: Context = {
systemPrompt: "You are a helpful assistant that can use tools to answer questions.",
messages: [
{
role: "user",
content: "Think about this briefly, then calculate 42 * 17 and 453 + 434 using the math_operation tool.",
timestamp: Date.now(),
},
],
tools: [calculatorTool],
};
let allTextContent = "";
let hasSeenThinking = false;
let hasSeenToolCalls = false;
const maxTurns = 5; // Prevent infinite loops
for (let turn = 0; turn < maxTurns; turn++) {
const response = await complete(model, context, options);
context.messages.push(response);
const results: ToolResultMessage[] = [];
for (const block of response.content) {
if (block.type === "text") {
allTextContent += block.text;
} else if (block.type === "thinking") {
hasSeenThinking = true;
} else if (block.type === "toolCall") {
hasSeenToolCalls = true;
expect(block.name).toBe("math_operation");
expect(block.id).toBeTruthy();
expect(block.arguments).toBeTruthy();
const { a, b, operation } = block.arguments;
let result: number;
switch (operation) {
case "add":
result = a + b;
break;
case "multiply":
result = a * b;
break;
default:
result = 0;
}
results.push({
role: "toolResult",
toolCallId: block.id,
toolName: block.name,
content: [{ type: "text", text: `${result}` }],
isError: false,
timestamp: Date.now(),
});
}
}
context.messages.push(...results);
expect(response.stopReason, `Error: ${response.errorMessage}`).not.toBe("error");
if (response.stopReason === "stop") {
break;
}
}
expect(hasSeenThinking || hasSeenToolCalls).toBe(true);
expect(allTextContent).toBeTruthy();
expect(allTextContent.includes("714")).toBe(true);
expect(allTextContent.includes("887")).toBe(true);
}
describe("Generate E2E Tests", () => {
describe.skipIf(!process.env.GEMINI_API_KEY)("Gemini Provider (gemini-2.5-flash)", () => {
const llm = getModel("google", "gemini-2.5-flash");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking", async () => {
await handleThinking(llm, { thinking: { enabled: true, budgetTokens: 1024 } });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { thinking: { enabled: true, budgetTokens: 2048 } });
});
it("should handle image input", async () => {
await handleImage(llm);
});
});
describe("Google Vertex Provider (gemini-3-flash-preview)", () => {
const vertexProject = process.env.GOOGLE_CLOUD_PROJECT || process.env.GCLOUD_PROJECT;
const vertexLocation = process.env.GOOGLE_CLOUD_LOCATION;
const vertexApiKey = process.env.GOOGLE_CLOUD_API_KEY;
const isVertexConfigured = Boolean(vertexProject && vertexLocation);
const vertexOptions = { project: vertexProject, location: vertexLocation } as const;
const llm = getModel("google-vertex", "gemini-3-flash-preview");
it.skipIf(!isVertexConfigured)("should complete basic text generation", async () => {
await basicTextGeneration(llm, vertexOptions);
});
it.skipIf(!vertexApiKey)("should complete basic text generation with Vertex API key", async () => {
await basicTextGeneration(llm, { apiKey: vertexApiKey! });
});
it.skipIf(!isVertexConfigured)("should handle tool calling", async () => {
await handleToolCall(llm, vertexOptions);
});
it.skipIf(!isVertexConfigured)("should handle thinking", async () => {
const { ThinkingLevel } = await import("@google/genai");
await handleThinking(llm, {
...vertexOptions,
thinking: { enabled: true, budgetTokens: 1024, level: ThinkingLevel.LOW },
});
});
it.skipIf(!isVertexConfigured)("should handle streaming", async () => {
await handleStreaming(llm, vertexOptions);
});
it.skipIf(!isVertexConfigured)("should handle multi-turn with thinking and tools", async () => {
const { ThinkingLevel } = await import("@google/genai");
await multiTurn(llm, {
...vertexOptions,
thinking: { enabled: true, budgetTokens: 1024, level: ThinkingLevel.MEDIUM },
});
});
it.skipIf(!isVertexConfigured)("should handle image input", async () => {
await handleImage(llm, vertexOptions);
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider (gpt-4o-mini)", () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini");
void _compat;
const llm: Model<"openai-completions"> = {
...baseModel,
api: "openai-completions",
};
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle image input", async () => {
await handleImage(llm);
});
});
describe.skipIf(!primeInferenceApiKey)("Prime Inference Provider (openai/gpt-5.5)", () => {
const llm = getModel("prime-inference", "openai/gpt-5.5");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle multi-turn with tools", async () => {
await multiTurn(llm);
});
});
describe.skipIf(!process.env.DEEPSEEK_API_KEY)("DeepSeek Provider (deepseek-flash via OpenAI Completions)", () => {
const llm = getModel("deepseek", "deepseek-flash");
it("should complete basic text generation", { retry: 3 }, async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", { retry: 3 }, async () => {
await handleToolCall(llm);
});
it("should handle streaming", { retry: 3 }, async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", { retry: 3 }, async () => {
await handleThinking(llm, { reasoningEffort: "high" });
});
it("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {
await multiTurn(llm, { reasoningEffort: "high" });
});
});
describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Responses Provider (gpt-5.4)", () => {
const llm = getModel("openai", "gpt-5.4");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking", async () => {
await handleThinking(llm, { reasoningEffort: "high" });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { reasoningEffort: "high" });
});
it("should handle image input", async () => {
await handleImage(llm);
});
});
describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic Provider (claude-haiku-4-5)", () => {
const model = getModel("anthropic", "claude-haiku-4-5");
it("should complete basic text generation", async () => {
await basicTextGeneration(model, { thinkingEnabled: true });
});
it("should handle tool calling", async () => {
await handleToolCall(model);
});
it("should handle streaming", async () => {
await handleStreaming(model);
});
it("should handle image input", async () => {
await handleImage(model);
});
});
describe.skipIf(!hasAzureOpenAICredentials())("Azure OpenAI Responses Provider (gpt-4o-mini)", () => {
const llm = getModel("azure-openai-responses", "gpt-4o-mini");
const azureDeploymentName = resolveAzureDeploymentName(llm.id);
const azureOptions = azureDeploymentName ? { azureDeploymentName } : {};
it("should complete basic text generation", async () => {
await basicTextGeneration(llm, azureOptions);
});
it("should handle tool calling", async () => {
await handleToolCall(llm, azureOptions);
});
it("should handle streaming", async () => {
await handleStreaming(llm, azureOptions);
});
it("should handle image input", async () => {
await handleImage(llm, azureOptions);
});
});
describe.skipIf(!process.env.XAI_API_KEY)("xAI Provider (grok-code-fast-1 via OpenAI Completions)", () => {
const llm = getModel("xai", "grok-code-fast-1");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, { reasoningEffort: "medium" });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { reasoningEffort: "medium" });
});
});
describe.skipIf(!process.env.GROQ_API_KEY)("Groq Provider (gpt-oss-20b via OpenAI Completions)", () => {
const llm = getModel("groq", "openai/gpt-oss-20b");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, { reasoningEffort: "medium" });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { reasoningEffort: "medium" });
});
});
describe.skipIf(!process.env.CEREBRAS_API_KEY)("Cerebras Provider (gpt-oss-120b via OpenAI Completions)", () => {
const llm = getModel("cerebras", "gpt-oss-120b");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, { reasoningEffort: "medium" });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { reasoningEffort: "medium" });
});
});
describe.skipIf(!hasCloudflareWorkersAICredentials())(
"Cloudflare Workers AI Provider (Kimi K2.6 via OpenAI Completions)",
() => {
const llm = getModel("cloudflare-workers-ai", "@cf/moonshotai/kimi-k2.6");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, { reasoningEffort: "medium" });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { reasoningEffort: "medium" });
});
},
);
describe.skipIf(!hasCloudflareAiGatewayCredentials() || !process.env.OPENAI_API_KEY)(
"Cloudflare AI Gateway → OpenAI BYOK (gpt-5.1 via /openai responses)",
() => {
const llm = getModel("cloudflare-ai-gateway", "gpt-5.1");
const options = { headers: { Authorization: `Bearer ${process.env.OPENAI_API_KEY}` } };
const thinkingOptions = {
...options,
thinkingEnabled: true,
reasoningEffort: "medium",
} satisfies StreamOptionsWithExtras;
it("should complete basic text generation", async () => {
await basicTextGeneration(llm, options);
});
it("should handle tool calling", async () => {
await handleToolCall(llm, options);
});
it("should handle streaming", async () => {
await handleStreaming(llm, options);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, thinkingOptions);
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, thinkingOptions);
});
},
);
describe.skipIf(!hasCloudflareAiGatewayCredentials() || !process.env.ANTHROPIC_API_KEY)(
"Cloudflare AI Gateway → Anthropic BYOK (Claude Sonnet 4.6 via /anthropic messages)",
() => {
const llm = getModels("cloudflare-ai-gateway").find((model) => model.name === "Claude Sonnet 4.6");
if (!llm) throw new Error("Cloudflare AI Gateway is missing Claude Sonnet 4.6");
const options = { headers: { Authorization: `Bearer ${process.env.ANTHROPIC_API_KEY}` } };
const thinkingOptions = {
...options,
thinkingEnabled: true,
reasoningEffort: "high",
} satisfies StreamOptionsWithExtras;
it("should complete basic text generation", async () => {
await basicTextGeneration(llm, options);
});
it("should handle tool calling", async () => {
await handleToolCall(llm, options);
});
it("should handle streaming", async () => {
await handleStreaming(llm, options);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, thinkingOptions);
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, thinkingOptions);
});
},
);
describe.skipIf(!process.env.HF_TOKEN)("Hugging Face Provider (Kimi-K2.5 via OpenAI Completions)", () => {
const llm = getModel("huggingface", "moonshotai/Kimi-K2.5");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, { reasoningEffort: "medium" });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { reasoningEffort: "medium" });
});
});
describe.skipIf(!process.env.OPENROUTER_API_KEY)("OpenRouter Provider (glm-4.5v via OpenAI Completions)", () => {
const llm = getModel("openrouter", "z-ai/glm-4.5v");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, { reasoningEffort: "medium" });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { reasoningEffort: "medium" });
});
it("should handle image input", async () => {
await handleImage(llm);
});
});
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)(
"Vercel AI Gateway Provider (google/gemini-2.5-flash via Anthropic Messages)",
() => {
const llm = getModel("vercel-ai-gateway", "google/gemini-2.5-flash");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle image input", async () => {
await handleImage(llm);
});
it("should handle multi-turn with tools", async () => {
await multiTurn(llm);
});
},
);
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)(
"Vercel AI Gateway Provider (anthropic/claude-opus-4.5 via Anthropic Messages)",
() => {
const llm = getModel("vercel-ai-gateway", "anthropic/claude-opus-4.5");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle image input", async () => {
await handleImage(llm);
});
it("should handle multi-turn with tools", async () => {
await multiTurn(llm);
});
},
);
describe.skipIf(!process.env.AI_GATEWAY_API_KEY)(
"Vercel AI Gateway Provider (openai/gpt-5.1-codex-max via Anthropic Messages)",
() => {
const llm = getModel("vercel-ai-gateway", "openai/gpt-5.1-codex-max");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle image input", async () => {
await handleImage(llm);
});
it("should handle multi-turn with tools", async () => {
await multiTurn(llm);
});
},
);
describe.skipIf(!process.env.ZAI_API_KEY)("zAI Provider (via OpenAI Completions)", () => {
const llm = getZaiTestModel({ toolStream: true });
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, { reasoningEffort: "medium" });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { reasoningEffort: "medium" });
});
it.skipIf(!llm.input.includes("image"))("should handle image input", async () => {
await handleImage(llm);
});
});
describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider (devstral-medium-latest)", () => {
const llm = getModel("mistral", "devstral-medium-latest");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
const llm = getModel("mistral", "magistral-medium-latest");
await handleThinking(llm, { promptMode: "reasoning" });
});
it("should handle multi-turn with thinking and tools", async () => {
const llm = getModel("mistral", "magistral-medium-latest");
await multiTurn(llm, { promptMode: "reasoning" });
});
});
describe.skipIf(!process.env.MISTRAL_API_KEY)("Mistral Provider (pixtral-12b with image support)", () => {
const llm = getModel("mistral", "pixtral-12b");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle image input", async () => {
await handleImage(llm);
});
});
describe.skipIf(!process.env.MINIMAX_API_KEY)("MiniMax Provider (MiniMax-M2.7 via Anthropic Messages)", () => {
const llm = getModel("minimax", "MiniMax-M2.7");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 });
});
});
describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider (Anthropic Messages)", () => {
const llm = getKimiCodingTestModel();
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 });
});
});
describe.skipIf(!process.env.XIAOMI_API_KEY)(
"Xiaomi MiMo (API billing) Provider (Xiaomi MiMo-V2.5-Pro via Anthropic Messages)",
() => {
const llm = getModel("xiaomi", "mimo-v2.5-pro");
const thinkingOptions = {
thinkingEnabled: true,
reasoningEffort: "high",
} satisfies StreamOptionsWithExtras;
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, thinkingOptions);
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, thinkingOptions);
});
},
);
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_CN_API_KEY)(
"Xiaomi MiMo Token Plan Provider (Xiaomi MiMo-V2.5-Pro via Anthropic Messages, CN region)",
() => {
const llm = getModel("xiaomi-token-plan-cn", "mimo-v2.5-pro");
const thinkingOptions = {
thinkingEnabled: true,
reasoningEffort: "high",
} satisfies StreamOptionsWithExtras;
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, thinkingOptions);
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, thinkingOptions);
});
},
);
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_AMS_API_KEY)(
"Xiaomi MiMo Token Plan Provider (Xiaomi MiMo-V2.5-Pro via Anthropic Messages, AMS region)",
() => {
const llm = getModel("xiaomi-token-plan-ams", "mimo-v2.5-pro");
const thinkingOptions = {
thinkingEnabled: true,
reasoningEffort: "high",
} satisfies StreamOptionsWithExtras;
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, thinkingOptions);
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, thinkingOptions);
});
},
);
describe.skipIf(!process.env.XIAOMI_TOKEN_PLAN_SGP_API_KEY)(
"Xiaomi MiMo Token Plan Provider (Xiaomi MiMo-V2.5-Pro via Anthropic Messages, SGP region)",
() => {
const llm = getModel("xiaomi-token-plan-sgp", "mimo-v2.5-pro");
const thinkingOptions = {
thinkingEnabled: true,
reasoningEffort: "high",
} satisfies StreamOptionsWithExtras;
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking mode", async () => {
await handleThinking(llm, thinkingOptions);
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, thinkingOptions);
});
},
);
describe("Anthropic OAuth Provider (claude-sonnet-4-6)", () => {
const model = getModel("anthropic", "claude-sonnet-4-6");
it.skipIf(!anthropicOAuthToken)("should complete basic text generation", async () => {
await basicTextGeneration(model, { apiKey: anthropicOAuthToken });
});
it.skipIf(!anthropicOAuthToken)("should handle tool calling", async () => {
await handleToolCall(model, { apiKey: anthropicOAuthToken });
});
it.skipIf(!anthropicOAuthToken)("should handle streaming", async () => {
await handleStreaming(model, { apiKey: anthropicOAuthToken });
});
it.skipIf(!anthropicOAuthToken)("should handle thinking", async () => {
await handleThinking(model, { apiKey: anthropicOAuthToken, thinkingEnabled: true });
});
it.skipIf(!anthropicOAuthToken)("should handle multi-turn with thinking and tools", async () => {
await multiTurn(model, { apiKey: anthropicOAuthToken, thinkingEnabled: true });
});
it.skipIf(!anthropicOAuthToken)("should handle image input", async () => {
await handleImage(model, { apiKey: anthropicOAuthToken });
});
});
describe("Anthropic OAuth Provider (claude-opus-4-6 with adaptive thinking)", () => {
const model = getModel("anthropic", "claude-opus-4-6");
it.skipIf(!anthropicOAuthToken)("should complete basic text generation", async () => {
await basicTextGeneration(model, { apiKey: anthropicOAuthToken });
});
it.skipIf(!anthropicOAuthToken)("should handle tool calling", async () => {
await handleToolCall(model, { apiKey: anthropicOAuthToken });
});
it.skipIf(!anthropicOAuthToken)("should handle streaming", async () => {
await handleStreaming(model, { apiKey: anthropicOAuthToken });
});
it.skipIf(!anthropicOAuthToken)("should handle adaptive thinking with effort high", async () => {
await handleThinking(model, { apiKey: anthropicOAuthToken, thinkingEnabled: true, effort: "high" });
});
it.skipIf(!anthropicOAuthToken)("should handle adaptive thinking with effort medium", async () => {
await handleThinking(model, { apiKey: anthropicOAuthToken, thinkingEnabled: true, effort: "medium" });
});
it.skipIf(!anthropicOAuthToken)("should handle multi-turn with adaptive thinking and tools", async () => {
await multiTurn(model, { apiKey: anthropicOAuthToken, thinkingEnabled: true, effort: "high" });
});
it.skipIf(!anthropicOAuthToken)("should handle image input", async () => {
await handleImage(model, { apiKey: anthropicOAuthToken });
});
});
describe("GitHub Copilot Provider (gpt-5.3-codex via OpenAI Completions)", () => {
const llm = getModel("github-copilot", "gpt-5.3-codex");
it.skipIf(!githubCopilotToken)("should complete basic text generation", async () => {
await basicTextGeneration(llm, { apiKey: githubCopilotToken });
});
it.skipIf(!githubCopilotToken)("should handle tool calling", async () => {
await handleToolCall(llm, { apiKey: githubCopilotToken });
});
it.skipIf(!githubCopilotToken)("should handle streaming", async () => {
await handleStreaming(llm, { apiKey: githubCopilotToken });
});
it.skipIf(!githubCopilotToken)("should handle thinking", async () => {
const thinkingModel = getModel("github-copilot", "gpt-5-mini");
await handleThinking(thinkingModel, { apiKey: githubCopilotToken, reasoningEffort: "high" });
});
it.skipIf(!githubCopilotToken)("should handle multi-turn with thinking and tools", async () => {
const thinkingModel = getModel("github-copilot", "gpt-5-mini");
await multiTurn(thinkingModel, { apiKey: githubCopilotToken, reasoningEffort: "high" });
});
it.skipIf(!githubCopilotToken)("should handle image input", async () => {
await handleImage(llm, { apiKey: githubCopilotToken });
});
});
describe("GitHub Copilot Provider (claude-sonnet-4 via Anthropic Messages)", () => {
const llm = getModel("github-copilot", "claude-sonnet-4.6");
it.skipIf(!githubCopilotToken)("should complete basic text generation", async () => {
await basicTextGeneration(llm, { apiKey: githubCopilotToken });
});
it.skipIf(!githubCopilotToken)("should handle tool calling", async () => {
await handleToolCall(llm, { apiKey: githubCopilotToken });
});
it.skipIf(!githubCopilotToken)("should handle streaming", async () => {
await handleStreaming(llm, { apiKey: githubCopilotToken });
});
it.skipIf(!githubCopilotToken)("should handle thinking", async () => {
await handleThinking(llm, { apiKey: githubCopilotToken, thinkingEnabled: true });
});
it.skipIf(!githubCopilotToken)("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { apiKey: githubCopilotToken, thinkingEnabled: true });
});
it.skipIf(!githubCopilotToken)("should handle image input", async () => {
await handleImage(llm, { apiKey: githubCopilotToken });
});
});
describe("OpenAI Codex Provider (gpt-5.4)", () => {
const llm = getModel("openai-codex", "gpt-5.4");
it.skipIf(!openaiCodexToken)("should complete basic text generation", async () => {
await basicTextGeneration(llm, { apiKey: openaiCodexToken });
});
it.skipIf(!openaiCodexToken)("should handle tool calling", async () => {
await handleToolCall(llm, { apiKey: openaiCodexToken });
});
it.skipIf(!openaiCodexToken)("should handle streaming", async () => {
await handleStreaming(llm, { apiKey: openaiCodexToken });
});
it.skipIf(!openaiCodexToken)("should handle thinking", async () => {
await handleThinking(llm, { apiKey: openaiCodexToken, reasoningEffort: "high" });
});
it.skipIf(!openaiCodexToken)("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { apiKey: openaiCodexToken });
});
it.skipIf(!openaiCodexToken)("should handle image input", async () => {
await handleImage(llm, { apiKey: openaiCodexToken });
});
});
describe("OpenAI Codex Provider (gpt-5.5)", () => {
const llm = getModel("openai-codex", "gpt-5.5");
it.skipIf(!openaiCodexToken)("should complete basic text generation", async () => {
await basicTextGeneration(llm, { apiKey: openaiCodexToken });
});
it.skipIf(!openaiCodexToken)("should handle tool calling", async () => {
await handleToolCall(llm, { apiKey: openaiCodexToken });
});
it.skipIf(!openaiCodexToken)("should handle streaming", async () => {
await handleStreaming(llm, { apiKey: openaiCodexToken });
});
it.skipIf(!openaiCodexToken)("should handle thinking with reasoningEffort xhigh", async () => {
await handleThinking(llm, { apiKey: openaiCodexToken, reasoningEffort: "xhigh" });
});
it.skipIf(!openaiCodexToken)("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { apiKey: openaiCodexToken, reasoningEffort: "xhigh" });
});
it.skipIf(!openaiCodexToken)("should handle image input", async () => {
await handleImage(llm, { apiKey: openaiCodexToken });
});
});
describe("OpenAI Codex Provider (gpt-5.5 via WebSocket)", () => {
const llm = getModel("openai-codex", "gpt-5.5");
const wsOptions = { apiKey: openaiCodexToken, transport: "websocket" as const };
it.skipIf(!openaiCodexToken)("should complete basic text generation", async () => {
await basicTextGeneration(llm, wsOptions);
});
it.skipIf(!openaiCodexToken)("should handle tool calling", async () => {
await handleToolCall(llm, wsOptions);
});
it.skipIf(!openaiCodexToken)("should handle streaming", async () => {
await handleStreaming(llm, wsOptions);
});
it.skipIf(!openaiCodexToken)("should handle thinking with reasoningEffort xhigh", async () => {
await handleThinking(llm, { ...wsOptions, reasoningEffort: "xhigh" });
});
it.skipIf(!openaiCodexToken)("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { ...wsOptions, reasoningEffort: "xhigh" });
});
it.skipIf(!openaiCodexToken)("should handle image input", async () => {
await handleImage(llm, wsOptions);
});
});
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider (claude-sonnet-4-5)", () => {
const llm = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
it("should complete basic text generation", async () => {
await basicTextGeneration(llm);
});
it("should handle tool calling", async () => {
await handleToolCall(llm);
});
it("should handle streaming", async () => {
await handleStreaming(llm);
});
it("should handle thinking", async () => {
await handleThinking(llm, { reasoning: "medium" });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { reasoning: "high" });
});
it("should handle image input", async () => {
await handleImage(llm);
});
});
describe.skipIf(!hasBedrockCredentials())("Amazon Bedrock Provider (claude-opus-4-6 interleaved thinking)", () => {
const llm = getModel("amazon-bedrock", "global.anthropic.claude-opus-4-6-v1");
it("should use adaptive thinking without anthropic_beta", async () => {
let capturedPayload: unknown;
const response = await complete(
llm,
{
systemPrompt: "You are a helpful assistant that uses tools when asked.",
messages: [
{
role: "user",
content: "Think first, then calculate 15 + 27 using the math_operation tool.",
timestamp: Date.now(),
},
],
tools: [calculatorTool],
},
{
reasoning: "xhigh",
interleavedThinking: true,
onPayload: (payload) => {
capturedPayload = payload;
},
},
);
expect(response.stopReason, `Error: ${response.errorMessage}`).not.toBe("error");
expect(capturedPayload).toBeTruthy();
const payload = capturedPayload as {
additionalModelRequestFields?: {
thinking?: { type?: string; display?: string };
output_config?: { effort?: string };
anthropic_beta?: string[];
};
};
expect(payload.additionalModelRequestFields?.thinking).toEqual({
type: "adaptive",
display: "summarized",
});
expect(payload.additionalModelRequestFields?.output_config).toEqual({ effort: "max" });
expect(payload.additionalModelRequestFields?.anthropic_beta).toBeUndefined();
});
it("should pass requestMetadata to the SDK payload", async () => {
const llmSonnet = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
let capturedPayload: unknown;
const metadata = { app: "pi-test", env: "ci" };
const response = await complete(
llmSonnet,
{
messages: [
{
role: "user",
content: "Say hi.",
timestamp: Date.now(),
},
],
},
{
requestMetadata: metadata,
onPayload: (payload) => {
capturedPayload = payload;
},
},
);
expect(response.stopReason, `Error: ${response.errorMessage}`).not.toBe("error");
expect(capturedPayload).toBeTruthy();
expect((capturedPayload as { requestMetadata?: unknown }).requestMetadata).toEqual(metadata);
});
it("should omit requestMetadata from payload when not provided", async () => {
const llmSonnet = getModel("amazon-bedrock", "global.anthropic.claude-sonnet-4-5-20250929-v1:0");
let capturedPayload: unknown;
const response = await complete(
llmSonnet,
{
messages: [
{
role: "user",
content: "Say hi.",
timestamp: Date.now(),
},
],
},
{
onPayload: (payload) => {
capturedPayload = payload;
},
},
);
expect(response.stopReason, `Error: ${response.errorMessage}`).not.toBe("error");
expect(capturedPayload).toBeTruthy();
expect("requestMetadata" in (capturedPayload as object)).toBe(false);
});
});
let ollamaInstalled = false;
if (!process.env.PI_NO_LOCAL_LLM) {
try {
execSync("which ollama", { stdio: "ignore" });
ollamaInstalled = true;
} catch {
ollamaInstalled = false;
}
}
describe.skipIf(!ollamaInstalled)("Ollama Provider (gpt-oss-20b via OpenAI Completions)", () => {
let llm: Model<"openai-completions">;
let ollamaProcess: ChildProcess | null = null;
beforeAll(async () => {
try {
execSync("ollama list | grep -q 'gpt-oss:20b'", { stdio: "ignore" });
} catch {
console.log("Pulling gpt-oss:20b model for Ollama tests...");
try {
execSync("ollama pull gpt-oss:20b", { stdio: "inherit" });
} catch (_e) {
console.warn("Failed to pull gpt-oss:20b model, tests will be skipped");
return;
}
}
ollamaProcess = spawn("ollama", ["serve"], {
detached: false,
stdio: "ignore",
});
await new Promise<void>((resolve) => {
const checkServer = async () => {
try {
const response = await fetch("http://localhost:11434/api/tags");
if (response.ok) {
resolve();
} else {
setTimeout(checkServer, 500);
}
} catch {
setTimeout(checkServer, 500);
}
};
setTimeout(checkServer, 1000); // Initial delay
});
llm = {
id: "gpt-oss:20b",
api: "openai-completions",
provider: "ollama",
baseUrl: "http://localhost:11434/v1",
reasoning: true,
input: ["text"],
contextWindow: 128000,
maxTokens: 16000,
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
name: "Ollama GPT-OSS 20B",
};
}, 30000); // 30 second timeout for setup
afterAll(() => {
if (ollamaProcess) {
ollamaProcess.kill("SIGTERM");
ollamaProcess = null;
}
});
it("should complete basic text generation", async () => {
await basicTextGeneration(llm, { apiKey: "test" });
});
it("should handle tool calling", async () => {
await handleToolCall(llm, { apiKey: "test" });
});
it("should handle streaming", async () => {
await handleStreaming(llm, { apiKey: "test" });
});
it("should handle thinking mode", async () => {
await handleThinking(llm, { apiKey: "test", reasoningEffort: "medium" });
});
it("should handle multi-turn with thinking and tools", async () => {
await multiTurn(llm, { apiKey: "test", reasoningEffort: "medium" });
});
});
});