e2e_test.go

//go:build e2e

package llm

// End-to-end tests against live LLM providers. Tag-gated so they never
// build into CI or a plain `go test ./...`:
//
//	go test -tags e2e -run 'TestE2E' -timeout 15m -v .
//
// Credentials come from the process environment or a repo-root .env file
// (KEY=VALUE lines, optional `export ` prefix and double quotes), parsed by
// loadDotEnv. Neither the file nor any key is ever logged — the SDK
// guarantees API keys stay out of all error text. Each provider's tests
// skip cleanly when its key is absent, so the suite covers exactly the
// providers you have credentials for.
//
// Adding a provider = one e2eTarget entry below. Model defaults can be
// overridden per provider via <ID>_E2E_MODEL (e.g. OPENROUTER_E2E_MODEL).

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"
)

// loadDotEnv parses a .env file and sets any variable not already present
// in the process environment (environment wins over file). Values may be
// wrapped in single or double quotes; `KEY=VALUE` and `export KEY=VALUE`
// forms are accepted; `#` starts a comment.
func loadDotEnv(t *testing.T, path string) {
	t.Helper()
	b, err := os.ReadFile(path)
	if err != nil {
		return // no .env: environment only
	}
	for i, line := range strings.Split(string(b), "\n") {
		line = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "export "))
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		k, v, ok := strings.Cut(line, "=")
		if !ok {
			t.Logf(".env line %d: not KEY=VALUE, ignored", i+1)
			continue
		}
		k = strings.TrimSpace(k)
		v = strings.Trim(strings.TrimSpace(v), `"'`)
		if k == "" {
			continue
		}
		if _, exists := os.LookupEnv(k); !exists {
			if err := os.Setenv(k, v); err != nil {
				t.Fatalf("setenv %s: %v", k, err)
			}
		}
	}
}

// e2eEnvKey resolves one credential, skipping the caller when absent.
func e2eEnvKey(t *testing.T, env string) string {
	t.Helper()
	loadDotEnv(t, ".env")
	key := strings.TrimSpace(os.Getenv(env))
	if key == "" {
		t.Skipf("%s not set (env or .env); skipping", env)
	}
	return key
}

// e2eTarget is one live provider under test.
type e2eTarget struct {
	id          string // registry id (built-in) or custom provider id
	baseURL     string // "" = built-in registry entry via FromEnv
	format      Format // wire format for custom providers
	keyEnv      string // env var holding the API key
	model       string // default chat model (override: <ID>_E2E_MODEL)
	tools       bool   // provider reliably supports tool calling
	streamUsage bool   // provider reliably returns usage on streams
}

var e2eTargets = []e2eTarget{
	{id: "deepseek", baseURL: "", keyEnv: "DEEPSEEK_API_KEY", model: "deepseek-chat", tools: true, streamUsage: true},
	{id: "openrouter", baseURL: "https://openrouter.ai/api/v1", format: FormatOpenAI, keyEnv: "OPENROUTER_API_KEY", model: "openai/gpt-4o-mini", tools: true},
	{id: "zai", baseURL: "", keyEnv: "ZAI_API_KEY", model: "glm-5-turbo", tools: true},
}

// chatModel resolves the target's model: <ID>_E2E_MODEL beats the default.
func (tg e2eTarget) chatModel() string {
	if v := strings.TrimSpace(os.Getenv(strings.ToUpper(tg.id) + "_E2E_MODEL")); v != "" {
		return v
	}
	return tg.model
}

// chat builds a chat client against the live endpoint.
func (tg e2eTarget) chat(t *testing.T) *ChatClient {
	t.Helper()
	e2eEnvKey(t, tg.keyEnv)
	var sdk *SDK
	if tg.baseURL == "" {
		sdk = New(FromEnv()) // built-in registry: production discovery path
	} else {
		sdk = New(WithProvider(tg.id,
			WithFormat(tg.format),
			WithBaseURL(tg.baseURL),
			WithAPIKey(e2eEnvKey(t, tg.keyEnv)),
		))
	}
	cc, err := sdk.Chat(tg.id, tg.chatModel())
	if err != nil {
		t.Fatalf("Chat(%s, %s): %v", tg.id, tg.chatModel(), err)
	}
	return cc
}

// badKeyClient builds a client identical to the target's but with a
// deliberately invalid key (error-taxonomy probe).
func (tg e2eTarget) badKeyClient(t *testing.T) *ChatClient {
	t.Helper()
	e2eEnvKey(t, tg.keyEnv)
	var sdk *SDK
	if tg.baseURL == "" {
		sdk = New(WithProvider(tg.id, WithAPIKey("sk-e2e-invalid-probe")))
	} else {
		sdk = New(WithProvider(tg.id,
			WithFormat(tg.format),
			WithBaseURL(tg.baseURL),
			WithAPIKey("sk-e2e-invalid-probe"),
		))
	}
	cc, err := sdk.Chat(tg.id, tg.chatModel())
	if err != nil {
		t.Fatalf("Chat(%s): %v", tg.id, err)
	}
	return cc
}

func e2eCtx(t *testing.T, d time.Duration) context.Context {
	t.Helper()
	ctx, cancel := context.WithTimeout(context.Background(), d)
	t.Cleanup(cancel)
	return ctx
}

// TestE2EProviders runs the generic matrix (discovery, buffered, streaming,
// tools, bad-key) for every provider whose key is present.
func TestE2EProviders(t *testing.T) {
	for _, tg := range e2eTargets {
		tg := tg
		t.Run(tg.id, func(t *testing.T) {
			t.Run("discovery", func(t *testing.T) { tg.testDiscovery(t) })
			t.Run("buffered", func(t *testing.T) { tg.testBuffered(t) })
			t.Run("streaming", func(t *testing.T) { tg.testStreaming(t) })
			if tg.tools {
				t.Run("tools", func(t *testing.T) { tg.testTools(t) })
			}
			t.Run("badkey", func(t *testing.T) { tg.testBadKey(t) })
		})
	}
}

func (tg e2eTarget) testDiscovery(t *testing.T) {
	t.Helper()
	e2eEnvKey(t, tg.keyEnv)
	sdk := New(FromEnv())
	if tg.baseURL != "" {
		sdk = New(WithProvider(tg.id,
			WithFormat(tg.format),
			WithBaseURL(tg.baseURL),
			WithAPIKey(strings.TrimSpace(os.Getenv(tg.keyEnv))),
		))
	}
	p, err := sdk.Provider(tg.id)
	if err != nil {
		t.Fatal(err)
	}
	if !p.Authenticated() {
		t.Fatal("provider resolved but not authenticated")
	}
	models, err := p.ListModels(e2eCtx(t, 60*time.Second))
	if err != nil {
		t.Fatalf("ListModels: %v", err)
	}
	if len(models) == 0 {
		t.Fatal("expected at least one accessible model")
	}
	ids := make([]string, 0, len(models))
	for _, m := range models {
		if m.ID == "" {
			t.Errorf("model with empty ID: %+v", m)
		}
		ids = append(ids, m.ID)
	}
	if len(ids) > 15 {
		ids = ids[:15]
	}
	t.Logf("accessible models (%d total, first 15): %s", len(models), strings.Join(ids, ", "))
}

func (tg e2eTarget) testBuffered(t *testing.T) {
	t.Helper()
	cc := tg.chat(t)
	res, err := cc.Call(e2eCtx(t, 120*time.Second), &ChatRequest{
		Messages:  []Message{{Role: RoleUser, Content: "Reply with exactly: OK"}},
		MaxTokens: 20,
	})
	if err != nil {
		t.Fatalf("Call: %v", err)
	}
	if !strings.Contains(strings.ToUpper(res.Content), "OK") {
		t.Errorf("content = %q, want it to contain OK", res.Content)
	}
	if res.FinishReason != FinishStop {
		t.Errorf("finish = %q, want stop", res.FinishReason)
	}
	if res.Usage.PromptTokens == 0 || res.Usage.CompletionTokens == 0 {
		t.Errorf("usage = %+v, want live token counts", res.Usage)
	}
}

func (tg e2eTarget) testStreaming(t *testing.T) {
	t.Helper()
	cc := tg.chat(t)
	var content strings.Builder
	sawContent := false
	res, err := cc.CallStream(e2eCtx(t, 120*time.Second), &ChatRequest{
		Messages:  []Message{{Role: RoleUser, Content: "Count from 1 to 5, digits only."}},
		MaxTokens: 60,
	}, func(d Delta) error {
		if d.Kind == DeltaContent {
			sawContent = true
			content.WriteString(d.Text)
		}
		return nil
	})
	if err != nil {
		t.Fatalf("CallStream: %v", err)
	}
	if !sawContent {
		t.Error("no content deltas received")
	}
	if res.Content == "" {
		t.Fatal("empty result content")
	}
	if res.Content != content.String() {
		t.Errorf("folded deltas %q != final result %q", content.String(), res.Content)
	}
	if res.FinishReason != FinishStop {
		t.Errorf("finish = %q, want stop", res.FinishReason)
	}
	if tg.streamUsage && res.Usage.CompletionTokens == 0 {
		t.Errorf("streaming usage missing (stream_options.include_usage): %+v", res.Usage)
	}
}

func (tg e2eTarget) testTools(t *testing.T) {
	t.Helper()
	cc := tg.chat(t)
	weather := ToolDef{
		Name:        "get_weather",
		Description: "Current weather for a city",
		Parameters:  json.RawMessage(`{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}`),
	}
	city := "Lisbon"
	question := fmt.Sprintf("What's the weather in %s? Use the tool.", city)
	ctx := e2eCtx(t, 120*time.Second)

	// Round trip: tool call -> tool result -> final answer.
	res, err := cc.Call(ctx, &ChatRequest{
		Messages:  []Message{{Role: RoleUser, Content: question}},
		Tools:     []ToolDef{weather},
		MaxTokens: 200,
	})
	if err != nil {
		t.Fatalf("tool call: %v", err)
	}
	if len(res.ToolCalls) != 1 {
		t.Fatalf("want exactly 1 tool call, got %d (finish %q, content %q)", len(res.ToolCalls), res.FinishReason, res.Content)
	}
	tc := res.ToolCalls[0]
	if tc.Name != "get_weather" {
		t.Errorf("tool name = %q, want get_weather", tc.Name)
	}
	if !json.Valid([]byte(tc.Arguments)) {
		t.Errorf("arguments not valid JSON: %q", tc.Arguments)
	}

	follow := &ChatRequest{
		Messages: []Message{
			{Role: RoleUser, Content: question},
			{Role: RoleAssistant, Content: res.Content, ToolCalls: res.ToolCalls},
			{Role: RoleTool, ToolCallID: tc.ID, ToolName: tc.Name, Content: `{"temp_c":24,"condition":"sunny"}`},
		},
		Tools:     []ToolDef{weather},
		MaxTokens: 200,
	}
	res2, err := cc.Call(ctx, follow)
	if err != nil {
		t.Fatalf("follow-up: %v", err)
	}
	if res2.Content == "" {
		t.Fatal("model did not answer using the tool result")
	}

	// Streamed: argument fragments must assemble into a valid call.
	streamed, err := cc.CallStream(ctx, &ChatRequest{
		Messages:  []Message{{Role: RoleUser, Content: "What's the weather in Tokyo? Use the tool."}},
		Tools:     []ToolDef{weather},
		MaxTokens: 200,
	}, func(Delta) error { return nil })
	if err != nil {
		t.Fatalf("CallStream: %v", err)
	}
	if len(streamed.ToolCalls) != 1 {
		t.Fatalf("want exactly 1 streamed tool call, got %d", len(streamed.ToolCalls))
	}
	if !json.Valid([]byte(streamed.ToolCalls[0].Arguments)) {
		t.Errorf("streamed arguments not valid JSON: %q", streamed.ToolCalls[0].Arguments)
	}
}

func (tg e2eTarget) testBadKey(t *testing.T) {
	t.Helper()
	cc := tg.badKeyClient(t)
	_, err := cc.Call(e2eCtx(t, 30*time.Second), &ChatRequest{Messages: []Message{{Role: RoleUser, Content: "hi"}}})
	var ae *APIError
	if !errors.As(err, &ae) {
		t.Fatalf("err = %v (%T), want *APIError", err, err)
	}
	if ae.Status != http.StatusUnauthorized {
		t.Errorf("status = %d, want 401", ae.Status)
	}
	if ae.Retryable {
		t.Error("401 must not be retryable")
	}
	if strings.Contains(err.Error(), "sk-e2e-invalid-probe") {
		t.Error("error text leaked the API key")
	}
}

// ── deepseek-specific arms ───────────────────────────────────────────────

// deepseek-reasoner: reasoning content is captured and the answer is right.
func TestE2EReasonerThinking(t *testing.T) {
	tg := e2eTargets[0]
	cc := tg.chat(t)
	res, err := cc.Call(e2eCtx(t, 180*time.Second), &ChatRequest{
		Messages:  []Message{{Role: RoleUser, Content: "A clock shows 3:15. What is the angle in degrees between the hour and minute hands? Work it out, then answer with the number only."}},
		MaxTokens: 1000,
	})
	if err != nil {
		t.Fatalf("Call: %v", err)
	}
	if res.ReasoningContent == "" {
		// Provider-side behavior: DeepSeek elides reasoning_content for some
		// prompts/runs. The SDK capture path is unit-covered; here we only
		// probe, not assert.
		t.Log("provider returned no reasoning content (server-side elision)")
	}
	if !strings.Contains(res.Content, "7.5") {
		// Model IQ is not the SDK's contract; when the provider elides
		// reasoning, arithmetic riddles can come back wrong. The live
		// invariants are: call succeeds, content parses, finish is canonical.
		t.Logf("answer = %q (provider may be wrong without reasoning)", res.Content)
	}
	if res.FinishReason != FinishStop && res.FinishReason != FinishLength {
		t.Errorf("finish = %q, want stop or length", res.FinishReason)
	}
}

// Reasoner streaming: reasoning deltas flow through DeltaReasoning.
func TestE2EReasonerStreaming(t *testing.T) {
	tg := e2eTargets[0]
	cc := tg.chat(t)
	var sawReasoning bool
	res, err := cc.CallStream(e2eCtx(t, 180*time.Second), &ChatRequest{
		Messages:  []Message{{Role: RoleUser, Content: "A water lily patch doubles in size every day. It covers the whole lake on day 48. On which day was it half covered? Answer with the day number only."}},
		MaxTokens: 1000,
	}, func(d Delta) error {
		if d.Kind == DeltaReasoning && d.Text != "" {
			sawReasoning = true
		}
		return nil
	})
	if err != nil {
		t.Fatalf("CallStream: %v", err)
	}
	if !sawReasoning {
		t.Log("provider returned no reasoning deltas (server-side elision); capture path is unit-covered")
	}
	if !strings.Contains(res.Content, "24") {
		// Model IQ is not the SDK's contract (see the buffered reasoner note).
		t.Logf("answer = %q (provider may be wrong without reasoning)", res.Content)
	}
	if res.FinishReason != FinishStop && res.FinishReason != FinishLength {
		t.Errorf("finish = %q, want stop or length", res.FinishReason)
	}
}

// ── TTS arms ─────────────────────────────────────────────────────────────

// e2eTTSModel resolves the TTS model for a provider id: <ID>_TTS_E2E_MODEL
// beats the default (kept separate from the chat-model override).
func e2eTTSModel(t *testing.T, id, def string) string {
	t.Helper()
	if v := strings.TrimSpace(os.Getenv(strings.ToUpper(id) + "_TTS_E2E_MODEL")); v != "" {
		return v
	}
	return def
}

// TestE2ETTSOpenAI probes the OpenAI /audio/speech path against the live
// OpenAI endpoint (built-in registry, key via OPENAI_API_KEY). Asserts only
// SDK guarantees: the call succeeds and non-empty audio with a non-empty
// MIME type comes back. The audio's decodability is never asserted.
func TestE2ETTSOpenAI(t *testing.T) {
	const keyEnv = "OPENAI_API_KEY"
	key := e2eEnvKey(t, keyEnv)
	sdk := New(WithProvider("openai", WithAPIKey(key)))
	res, err := sdk.Speak(t.Context(), "openai", e2eTTSModel(t, "openai", "tts-1"), SpeakRequest{
		Text:  "go-llm-sdk text to speech probe.",
		Voice: "alloy",
	})
	if err != nil {
		t.Fatalf("Speak: %v", err)
	}
	if len(res.Audio) == 0 {
		t.Errorf("Speak returned empty audio")
	}
	if res.MIMEType == "" {
		t.Errorf("Speak returned empty MIMEType")
	}
	t.Logf("audio=%d bytes mime=%q model=%q", len(res.Audio), res.MIMEType, res.Model)
}

// TestE2ETTSGemini probes the Gemini AUDIO-modality TTS path (built-in
// registry, key via GEMINI_API_KEY). Same soft contract: non-empty audio
// bytes + non-empty MIME type; no format assumptions beyond what the SDK
// already guarantees.
func TestE2ETTSGemini(t *testing.T) {
	const keyEnv = "GEMINI_API_KEY"
	key := e2eEnvKey(t, keyEnv)
	sdk := New(WithProvider("gemini", WithAPIKey(key)))
	res, err := sdk.Speak(t.Context(), "gemini", e2eTTSModel(t, "gemini", "gemini-2.5-flash-preview-tts"), SpeakRequest{
		Text:  "go-llm-sdk text to speech probe.",
		Voice: "Kore",
	})
	if err != nil {
		t.Fatalf("Speak: %v", err)
	}
	if len(res.Audio) == 0 {
		t.Errorf("Speak returned empty audio")
	}
	if res.MIMEType == "" {
		t.Errorf("Speak returned empty MIMEType")
	}
	t.Logf("audio=%d bytes mime=%q model=%q", len(res.Audio), res.MIMEType, res.Model)
}

// TestE2ETranscribeOpenAI probes the OpenAI /audio/transcriptions path
// against the live endpoint (built-in registry, key via OPENAI_API_KEY).
// Self-contained: audio is synthesized first via Speak, then transcribed
// (model override via OPENAI_STT_E2E_MODEL, default whisper-1). Asserts
// only SDK guarantees: the call succeeds and non-empty text comes back.
// Transcript accuracy is never asserted.
func TestE2ETranscribeOpenAI(t *testing.T) {
	const keyEnv = "OPENAI_API_KEY"
	key := e2eEnvKey(t, keyEnv)
	sdk := New(WithProvider("openai", WithAPIKey(key)))
	spoken, err := sdk.Speak(t.Context(), "openai", e2eTTSModel(t, "openai", "tts-1"), SpeakRequest{
		Text:  "go-llm-sdk speech to text probe.",
		Voice: "alloy",
	})
	if err != nil {
		t.Fatalf("Speak: %v", err)
	}
	sttModel := "whisper-1"
	if v := strings.TrimSpace(os.Getenv("OPENAI_STT_E2E_MODEL")); v != "" {
		sttModel = v
	}
	res, err := sdk.Transcribe(t.Context(), "openai", sttModel, TranscribeRequest{
		Audio:    spoken.Audio,
		Filename: "probe.mp3",
	})
	if err != nil {
		t.Fatalf("Transcribe: %v", err)
	}
	if res.Text == "" {
		t.Errorf("Transcribe returned empty text")
	}
	t.Logf("text=%q duration=%.1fs model=%q", res.Text, res.DurationSec, res.Model)
}