Add modelproxytest with the replies a fake model upstream streams to a real CLI

README.md

@@ -334,6 +334,12 @@ without conversation text.
 `NewCertificate` returns a throwaway CA and a server certificate for the HTTPS
 listener the `ChatGPT` provider needs.
 
+`modelproxy/modelproxytest` writes the replies a fake upstream needs to drive a
+real CLI through the proxy in a test: `AnthropicText` and `AnthropicToolUse`
+stream a Messages API turn, `ResponsesFunctionCall` streams a Responses API
+function call, and `CodexStartup` answers the ChatGPT routes Codex calls before
+its first model request when it signs in with an access token.
+
 ## License
 
 MIT

modelproxy/modelproxytest/modelproxytest.go

@@ -0,0 +1,103 @@
+// Package modelproxytest writes the model API responses an agent CLI accepts, so a test can stand a fake
+// upstream behind modelproxy and drive a real CLI without a model key.
+package modelproxytest
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync/atomic"
+)
+
+// messages numbers the Messages API replies, since ClaudeHarness counts the usage of each message ID once.
+var messages atomic.Int64
+
+// AnthropicText streams a Messages API reply made of one text block, which ends the turn.
+//
+//nolint:goconst // protocol field names are data
+func AnthropicText(w http.ResponseWriter, model, text string) {
+	anthropic(w, model, "end_turn",
+		map[string]any{"type": "text", "text": ""},
+		map[string]any{"type": "text_delta", "text": text})
+}
+
+// AnthropicToolUse streams a Messages API reply that calls the tool name with input, a JSON object. Each call
+// in one conversation needs its own id.
+//
+//nolint:goconst // protocol field names are data
+func AnthropicToolUse(w http.ResponseWriter, model, id, name, input string) {
+	anthropic(w, model, "tool_use",
+		map[string]any{"type": "tool_use", "id": id, "name": name, "input": map[string]any{}},
+		map[string]any{"type": "input_json_delta", "partial_json": input})
+}
+
+//nolint:goconst // protocol field names are data
+func anthropic(w http.ResponseWriter, model, stop string, block, delta map[string]any) {
+	stream := newStream(w)
+	stream.event("message_start", map[string]any{"type": "message_start", "message": map[string]any{
+		"id": "msg_" + strconv.FormatInt(messages.Add(1), 10), "type": "message", "role": "assistant", "model": model, "content": []any{},
+		"stop_reason": nil, "stop_sequence": nil, "usage": map[string]int{"input_tokens": 1, "output_tokens": 0},
+	}})
+	stream.event("content_block_start", map[string]any{"type": "content_block_start", "index": 0, "content_block": block})
+	stream.event("content_block_delta", map[string]any{"type": "content_block_delta", "index": 0, "delta": delta})
+	stream.event("content_block_stop", map[string]any{"type": "content_block_stop", "index": 0})
+	stream.event("message_delta", map[string]any{"type": "message_delta", "delta": map[string]any{"stop_reason": stop, "stop_sequence": nil}, "usage": map[string]int{"output_tokens": 1}})
+	stream.event("message_stop", map[string]any{"type": "message_stop"})
+}
+
+// ResponsesFunctionCall streams a Responses API reply that calls the function name with arguments, a JSON
+// object, as Codex expects.
+//
+//nolint:goconst // protocol field names are data
+func ResponsesFunctionCall(w http.ResponseWriter, model, callID, name, arguments string) {
+	item := map[string]any{"type": "function_call", "id": "fc_" + callID, "call_id": callID, "name": name, "arguments": arguments}
+	stream := newStream(w)
+	stream.event("response.created", map[string]any{"type": "response.created", "response": map[string]any{"id": "resp_fixture", "status": "in_progress", "output": []any{}}})
+	stream.event("response.output_item.done", map[string]any{"type": "response.output_item.done", "output_index": 0, "item": item})
+	stream.event("response.completed", map[string]any{"type": "response.completed", "response": map[string]any{
+		"id": "resp_fixture", "status": "completed", "model": model, "output": []any{item},
+		"usage": map[string]int{"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
+	}})
+}
+
+// CodexStartup answers the ChatGPT routes Codex calls before its first model request when it signs in with
+// an access token, and reports whether r was one of them.
+func CodexStartup(w http.ResponseWriter, r *http.Request) bool {
+	var body string
+	switch {
+	case strings.HasSuffix(r.URL.Path, "/v1/user-auth-credential/whoami"):
+		body = `{"email":"agent@example.com","chatgpt_user_id":"user-fixture","chatgpt_account_id":"account-fixture","chatgpt_plan_type":"enterprise","chatgpt_account_is_fedramp":false}`
+	case strings.HasSuffix(r.URL.Path, "/wham/config/bundle"):
+		body = `{}`
+	case strings.HasSuffix(r.URL.Path, "/codex/models"):
+		body = `{"models":[]}`
+	default:
+		return false
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_, _ = io.WriteString(w, body)
+	return true
+}
+
+type stream struct {
+	w  http.ResponseWriter
+	rc *http.ResponseController
+}
+
+func newStream(w http.ResponseWriter) stream {
+	w.Header().Set("Content-Type", "text/event-stream")
+	w.WriteHeader(http.StatusOK)
+	return stream{w, http.NewResponseController(w)}
+}
+
+func (s stream) event(kind string, data any) {
+	raw, err := json.Marshal(data)
+	if err != nil {
+		panic(err)
+	}
+	_, _ = fmt.Fprintf(s.w, "event: %s\ndata: %s\n\n", kind, raw)
+	_ = s.rc.Flush()
+}

modelproxy/modelproxytest/modelproxytest_test.go

@@ -0,0 +1,124 @@
+package modelproxytest
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+type sse struct {
+	kind string
+	data map[string]any
+}
+
+func events(t *testing.T, write func(http.ResponseWriter)) []sse {
+	t.Helper()
+	rec := httptest.NewRecorder()
+	write(rec)
+	if got := rec.Header().Get("Content-Type"); got != "text/event-stream" {
+		t.Fatalf("Content-Type = %q", got)
+	}
+	var out []sse
+	for _, chunk := range strings.Split(strings.TrimSpace(rec.Body.String()), "\n\n") {
+		kind, data, ok := strings.Cut(chunk, "\ndata: ")
+		if !ok || !strings.HasPrefix(kind, "event: ") {
+			t.Fatalf("malformed event %q", chunk)
+		}
+		var value map[string]any
+		if err := json.Unmarshal([]byte(data), &value); err != nil {
+			t.Fatalf("event data %q: %v", data, err)
+		}
+		if value["type"] != strings.TrimPrefix(kind, "event: ") {
+			t.Fatalf("event %q carries type %v", kind, value["type"])
+		}
+		out = append(out, sse{strings.TrimPrefix(kind, "event: "), value})
+	}
+	return out
+}
+
+func kinds(events []sse) string {
+	names := make([]string, len(events))
+	for i, e := range events {
+		names[i] = e.kind
+	}
+	return strings.Join(names, ",")
+}
+
+func TestAnthropicText(t *testing.T) {
+	got := events(t, func(w http.ResponseWriter) { AnthropicText(w, "claude-test", "Done.") })
+	if kinds(got) != "message_start,content_block_start,content_block_delta,content_block_stop,message_delta,message_stop" {
+		t.Fatalf("events = %s", kinds(got))
+	}
+	if delta := got[2].data["delta"].(map[string]any); delta["text"] != "Done." {
+		t.Errorf("delta = %v", delta)
+	}
+	if delta := got[4].data["delta"].(map[string]any); delta["stop_reason"] != "end_turn" {
+		t.Errorf("message delta = %v", delta)
+	}
+}
+
+func TestAnthropicToolUse(t *testing.T) {
+	got := events(t, func(w http.ResponseWriter) {
+		AnthropicToolUse(w, "claude-test", "tool_1", "Bash", `{"command":"true"}`)
+	})
+	block := got[1].data["content_block"].(map[string]any)
+	if block["type"] != "tool_use" || block["id"] != "tool_1" || block["name"] != "Bash" {
+		t.Errorf("block = %v", block)
+	}
+	if delta := got[2].data["delta"].(map[string]any); delta["partial_json"] != `{"command":"true"}` {
+		t.Errorf("delta = %v", delta)
+	}
+	if delta := got[4].data["delta"].(map[string]any); delta["stop_reason"] != "tool_use" {
+		t.Errorf("message delta = %v", delta)
+	}
+}
+
+func TestAnthropicRepliesHaveTheirOwnMessageID(t *testing.T) {
+	first := events(t, func(w http.ResponseWriter) {
+		AnthropicToolUse(w, "claude-test", "tool_1", "Bash", `{"command":"true"}`)
+	})
+	second := events(t, func(w http.ResponseWriter) { AnthropicText(w, "claude-test", "Done.") })
+	id := func(got []sse) any { return got[0].data["message"].(map[string]any)["id"] }
+	if id(first) == nil || id(first) == id(second) {
+		t.Errorf("message IDs = %v, %v", id(first), id(second))
+	}
+}
+
+func TestResponsesFunctionCall(t *testing.T) {
+	got := events(t, func(w http.ResponseWriter) {
+		ResponsesFunctionCall(w, "gpt-test", "call_1", "exec_command", `{"cmd":"true"}`)
+	})
+	if kinds(got) != "response.created,response.output_item.done,response.completed" {
+		t.Fatalf("events = %s", kinds(got))
+	}
+	item := got[1].data["item"].(map[string]any)
+	if item["call_id"] != "call_1" || item["name"] != "exec_command" || item["arguments"] != `{"cmd":"true"}` {
+		t.Errorf("item = %v", item)
+	}
+	if response := got[2].data["response"].(map[string]any); response["status"] != "completed" || response["model"] != "gpt-test" {
+		t.Errorf("response = %v", response)
+	}
+}
+
+func TestCodexStartup(t *testing.T) {
+	for path, want := range map[string]bool{
+		"/model-proxy/auth/v1/user-auth-credential/whoami": true,
+		"/backend-api/wham/config/bundle":                  true,
+		"/backend-api/codex/models":                        true,
+		"/backend-api/codex/responses":                     false,
+	} {
+		rec := httptest.NewRecorder()
+		if got := CodexStartup(rec, httptest.NewRequest(http.MethodGet, path, nil)); got != want {
+			t.Errorf("CodexStartup(%s) = %v", path, got)
+			continue
+		}
+		if want && !json.Valid(rec.Body.Bytes()) {
+			t.Errorf("%s body = %q", path, rec.Body.String())
+		}
+		if !want && rec.Body.Len() != 0 {
+			t.Errorf("%s wrote %q", path, rec.Body.String())
+		}
+	}
+}