From c54ae9bc1df1e309fdd97802cbec73904d917abf Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 24 Sep 2026 13:06:30 +0800 Subject: [PATCH] fix(upstreams): show cached models when opening editor Read model rows from the saved upstream snapshot on editor entry without waiting for discovery. Keep Custom discovery alongside the routable cache so overridden and rerank rows remain editable after reload. Remove unused editor loader requests and initial error state, cover cached/cold/empty catalog behavior, and make two WebSocket tests use their existing mocked upstream. --- .../upstream-editor/page-fetch_test.tsx | 37 +++++++- .../upstream-editor/page-yaml-submit_test.tsx | 5 +- ...dashboard-providers-upstreams-edit_test.ts | 41 ++++++++- .../src/components/upstream-editor/data.ts | 22 ++--- .../src/components/upstream-editor/page.tsx | 15 +-- .../dashboard-providers-upstreams-copy.tsx | 9 +- .../dashboard-providers-upstreams-edit.tsx | 3 +- .../dashboard-providers-upstreams-new.tsx | 8 +- .../control-plane/upstreams/routes_test.ts | 91 +++++++++++++++++++ .../chat/openai-responses/websocket_test.ts | 8 +- packages/gateway/__tests__/repo/memory.ts | 4 +- .../__tests__/repo/upstream-codecs_test.ts | 27 ++++++ .../upstreams/models-cache-projection.ts | 33 +++++++ .../upstreams/models-cache-status.ts | 11 --- .../src/control-plane/upstreams/models.ts | 18 +--- .../src/control-plane/upstreams/routes.ts | 14 +-- .../gateway/src/execution/models-refresh.ts | 9 +- packages/gateway/src/repo/upstream-codecs.ts | 18 ++++ packages/provider/src/model.ts | 5 +- 19 files changed, 292 insertions(+), 86 deletions(-) create mode 100644 packages/gateway/__tests__/repo/upstream-codecs_test.ts create mode 100644 packages/gateway/src/control-plane/upstreams/models-cache-projection.ts delete mode 100644 packages/gateway/src/control-plane/upstreams/models-cache-status.ts diff --git a/apps/web/__tests__/components/upstream-editor/page-fetch_test.tsx b/apps/web/__tests__/components/upstream-editor/page-fetch_test.tsx index 0e142ab8f5..88c2ff958d 100644 --- a/apps/web/__tests__/components/upstream-editor/page-fetch_test.tsx +++ b/apps/web/__tests__/components/upstream-editor/page-fetch_test.tsx @@ -10,6 +10,7 @@ import { UpstreamEditorPage } from '../../../src/components/upstream-editor/page import { i18n } from '../../../src/i18n'; import { upstreamRecord } from '../../api/upstream-fixture'; import { renderInApp } from '../../render'; +import type { UpstreamModelConfig } from '@floway-dev/provider/model-config'; const apiMocks = vi.hoisted(() => ({ patch: vi.fn(), listModels: vi.fn(), previewModels: vi.fn() })); @@ -74,14 +75,14 @@ const customRecord = upstreamRecord('up_custom', { }, state: null, }); -const discovered = [{ upstreamModelId: 'new-model', publicModelId: 'new-model', endpoints: { openaiChatCompletions: {} } }]; +const discovered = [{ upstreamModelId: 'new-model', publicModelId: 'new-model', kind: 'chat' as const, endpoints: { openaiChatCompletions: {} } }]; -const renderPage = (currentRecord = record) => { +const renderPage = (currentRecord = record, initialDiscovered: UpstreamModelConfig[] | null = null) => { const router = createMemoryRouter([{ path: '/editor', element: , }], { initialEntries: ['/editor'] }); return renderInApp(); @@ -94,6 +95,34 @@ beforeEach(() => { apiMocks.previewModels.mockResolvedValue({ data: { kind: 'ollama', data: [] }, error: null }); }); +test('opening an editor shows the stored model snapshot without fetching', () => { + renderPage(record, discovered); + expect(screen.getByTestId('discovered').textContent).toBe('new-model'); + expect(screen.getByTestId('catalog-available').textContent).toBe('true'); + expect(apiMocks.listModels).not.toHaveBeenCalled(); +}); + +test('a successful empty snapshot is still available', () => { + renderPage(record, []); + expect(screen.getByTestId('discovered').textContent).toBe(''); + expect(screen.getByTestId('catalog-available').textContent).toBe('true'); + expect(apiMocks.listModels).not.toHaveBeenCalled(); +}); + +test('failed refresh keeps a previously loaded catalog available', async () => { + apiMocks.listModels.mockResolvedValue({ + error: { + message: 'HTTP 503: unavailable', + raw: { modelsCache: { fetchedAt: 100, modelCount: 1, lastError: { message: 'HTTP 503: unavailable', at: 200 } } }, + }, + }); + renderPage(record, discovered); + fireEvent.click(screen.getByRole('button', { name: 'Fetch models' })); + await waitFor(() => expect(screen.getByTestId('models-error').textContent).toBe('HTTP 503: unavailable')); + expect(screen.getByTestId('discovered').textContent).toBe('new-model'); + expect(screen.getByTestId('catalog-available').textContent).toBe('true'); +}); + test('metadata-only OAuth edits fetch the saved record and keep form changes unsaved', async () => { renderPage(); expect(screen.getByTestId('catalog-available').textContent).toBe('false'); diff --git a/apps/web/__tests__/components/upstream-editor/page-yaml-submit_test.tsx b/apps/web/__tests__/components/upstream-editor/page-yaml-submit_test.tsx index 5a0b408efc..8ae8ae41c8 100644 --- a/apps/web/__tests__/components/upstream-editor/page-yaml-submit_test.tsx +++ b/apps/web/__tests__/components/upstream-editor/page-yaml-submit_test.tsx @@ -70,14 +70,11 @@ const renderPage = () => { const router = createMemoryRouter([{ path: '/editor', element: , }], { initialEntries: ['/editor?view=yaml'] }); return renderInApp(); diff --git a/apps/web/__tests__/routes/dashboard-providers-upstreams-edit_test.ts b/apps/web/__tests__/routes/dashboard-providers-upstreams-edit_test.ts index d17aa08c5c..640cbacdcb 100644 --- a/apps/web/__tests__/routes/dashboard-providers-upstreams-edit_test.ts +++ b/apps/web/__tests__/routes/dashboard-providers-upstreams-edit_test.ts @@ -16,15 +16,46 @@ vi.mock('../../src/components/upstream-editor/data', async importOriginal => ({ beforeEach(() => { vi.clearAllMocks(); - mocks.get.mockResolvedValue({ data: { id: 'up_saved', modelsCache: { fetchedAt: 100, lastError: null, modelCount: 2 } }, error: null }); - mocks.loadAux.mockResolvedValue({ proxies: [], backoffs: [], upstreams: [], runtime: { kind: 'node', runtimeLocation: 'TEST' } }); + mocks.get.mockResolvedValue({ + data: { + id: 'up_saved', + modelsCache: { fetchedAt: 100, lastError: null, modelCount: 2 }, + cachedModels: [{ upstreamModelId: 'cached-model', publicModelId: 'cached-model', kind: 'chat', endpoints: { openaiResponses: {} } }], + }, error: null, + }); + mocks.loadAux.mockResolvedValue({ proxies: [], runtime: { kind: 'node', runtimeLocation: 'TEST' } }); }); -test('opening a saved upstream reads its record and cache status without fetching models', async () => { +test('opening a saved upstream reads cached model rows without fetching models', async () => { const loaded = await clientLoader({ params: { id: 'up_saved' } } as Parameters[0]); expect(loaded.record.modelsCache).toEqual({ fetchedAt: 100, lastError: null, modelCount: 2 }); - expect(loaded.discovered).toEqual([]); - expect(loaded.modelsError).toBeNull(); + expect(loaded.discovered?.map(model => model.upstreamModelId)).toEqual(['cached-model']); expect(mocks.get).toHaveBeenCalledTimes(1); expect(mocks.listModels).not.toHaveBeenCalled(); }); + +test('opening an upstream without a successful cache keeps discovery unavailable', async () => { + mocks.get.mockResolvedValue({ + data: { + id: 'up_saved', + modelsCache: { fetchedAt: null, lastError: null, modelCount: null }, + cachedModels: null, + }, error: null, + }); + const loaded = await clientLoader({ params: { id: 'up_saved' } } as Parameters[0]); + expect(loaded.discovered).toBeNull(); + expect(mocks.listModels).not.toHaveBeenCalled(); +}); + +test('a successful empty cache remains available without a refresh', async () => { + mocks.get.mockResolvedValue({ + data: { + id: 'up_saved', + modelsCache: { fetchedAt: 100, lastError: null, modelCount: 0 }, + cachedModels: [], + }, error: null, + }); + const loaded = await clientLoader({ params: { id: 'up_saved' } } as Parameters[0]); + expect(loaded.discovered).toEqual([]); + expect(mocks.listModels).not.toHaveBeenCalled(); +}); diff --git a/apps/web/src/components/upstream-editor/data.ts b/apps/web/src/components/upstream-editor/data.ts index 7b88e9ada8..ff760f17e6 100644 --- a/apps/web/src/components/upstream-editor/data.ts +++ b/apps/web/src/components/upstream-editor/data.ts @@ -3,7 +3,6 @@ import type { InferRequestType } from 'hono/client'; import { PATH_OVERRIDE_PATHS } from './endpoints'; import { api, callApi } from '../../api/client'; import type { - BackoffRow, ProxyRecord, ProviderModelsFailureResponse, UpstreamRecord, @@ -23,15 +22,12 @@ export interface RuntimeInfo { export interface EditorAuxData { proxies: ProxyRecord[]; - backoffs: BackoffRow[]; runtime: RuntimeInfo; - upstreams: UpstreamRecord[]; } interface UpstreamEditorLoaderDataBase extends EditorAuxData { record: UpstreamRecord; - discovered: UpstreamModelConfig[]; - modelsError: ModelListingFailure | null; + discovered: UpstreamModelConfig[] | null; // For providers whose credential is an API key, the editor can keep that // stored key when the form's blank secret field is left alone; OAuth // providers do not copy their tokens and start from the blueprint @@ -88,19 +84,15 @@ export const providerDefaultName: Record = { }; export const loadEditorAux = async (): Promise => { - const [proxies, backoffs, runtime, upstreams] = await Promise.all([ + const [proxies, runtime] = await Promise.all([ callApi(() => api.api.proxies.$get()), - callApi(() => api.api.proxies.backoffs.$get()), callApi(() => api.api['runtime-info'].$get()), - callApi(() => api.api.upstreams.$get()), ]); - const error = proxies.error ?? backoffs.error ?? runtime.error ?? upstreams.error; - if (error) throw new Error(error.message); + if (proxies.error) throw new Error(proxies.error.message); + if (runtime.error) throw new Error(runtime.error.message); return { - proxies: proxies.data!, - backoffs: backoffs.data!, - runtime: runtime.data!, - upstreams: upstreams.data!, + proxies: proxies.data, + runtime: runtime.data, }; }; @@ -130,7 +122,7 @@ export const manualModelsSupported = (record: UpstreamRecord): record is Extract record.kind === 'custom' || record.kind === 'azure' || record.kind === 'ollama'; export interface ModelCatalogFetch { - /** Null when nothing was listed, which leaves whatever the caller already shows. */ + /** Null when the request produced no new catalog. */ discovered: UpstreamModelConfig[] | null; modelsError: ModelListingFailure | null; modelsCache: UpstreamRecord['modelsCache'] | null; diff --git a/apps/web/src/components/upstream-editor/page.tsx b/apps/web/src/components/upstream-editor/page.tsx index f6282bc3d7..aa8ae890d8 100644 --- a/apps/web/src/components/upstream-editor/page.tsx +++ b/apps/web/src/components/upstream-editor/page.tsx @@ -61,8 +61,8 @@ export function UpstreamEditorPage({ data }: { data: UpstreamEditorLoaderData }) setRecord(next); }, []); const [discovered, setDiscovered] = useState(data.discovered); - const [catalogAvailable, setCatalogAvailable] = useState(false); - const [modelsError, setModelsError] = useState(data.modelsError); + const catalogAvailable = discovered !== null; + const [modelsError, setModelsError] = useState(null); const [saveError, setSaveError] = useState(null); const [saving, setSaving] = useState(false); const [modelsYamlDraft, setModelsYamlDraft] = useState(null); @@ -166,8 +166,9 @@ export function UpstreamEditorPage({ data }: { data: UpstreamEditorLoaderData }) : await previewDraftModelCatalog(record, getValues(), { signal }); if (signal.aborted) return; setModelsError(catalog.modelsError); - setCatalogAvailable(catalog.discovered !== null && catalog.modelsError === null); - if (catalog.discovered) setDiscovered(catalog.discovered); + if (catalog.discovered !== null) { + setDiscovered(catalog.discovered); + } else if (catalog.modelsCache?.fetchedAt === null) setDiscovered(null); if (catalog.modelsCache) updateRecord({ ...recordRef.current, modelsCache: catalog.modelsCache } as UpstreamRecord); }, [discoveryInputsDirty, getValues, record, updateRecord])); @@ -232,7 +233,7 @@ export function UpstreamEditorPage({ data }: { data: UpstreamEditorLoaderData }) if (invalidatesPendingFetch) cancelModelsRefresh(); updateRecord(saved); reset(valuesFromRecord(saved)); - if (invalidatesDiscovered) { setModelsError(null); setDiscovered([]); setCatalogAvailable(false); } + if (invalidatesDiscovered) { setModelsError(null); setDiscovered(null); } handle.succeed(t('dashboard.upstreamEditor.toast.saved')); savedRecord = saved; // `saving` stays set during the route handoff; the old form is still @@ -271,7 +272,7 @@ export function UpstreamEditorPage({ data }: { data: UpstreamEditorLoaderData }) - + diff --git a/apps/web/src/routes/dashboard-providers-upstreams-copy.tsx b/apps/web/src/routes/dashboard-providers-upstreams-copy.tsx index 00ac09bacc..adf4d4f61b 100644 --- a/apps/web/src/routes/dashboard-providers-upstreams-copy.tsx +++ b/apps/web/src/routes/dashboard-providers-upstreams-copy.tsx @@ -82,26 +82,27 @@ const copyableRecord = (source: UpstreamRecord, name: string, hue: number): { export async function clientLoader({ params }: Route.ClientLoaderArgs) { await requireDashboardAdmin(); - const [sourceResult, aux] = await Promise.all([ + const [sourceResult, aux, upstreamsResult] = await Promise.all([ callApi(() => api.api.upstreams[':id'].$get({ param: { id: params.id } })), loadEditorAux(), + callApi(() => api.api.upstreams.$get()), ]); if (sourceResult.error?.status === 404) { throw redirect('/dashboard/providers/upstreams?missing=1'); } if (sourceResult.error) throw new Error(sourceResult.error.message); + if (upstreamsResult.error) throw new Error(upstreamsResult.error.message); const source = sourceResult.data; const { record, preserveCredentials } = copyableRecord( source, i18n.t('dashboard.upstreams.copy.nameSuffix', { name: source.name }), - pickDistinctHue(aux.upstreams.map(upstream => upstream.hue)), + pickDistinctHue(upstreamsResult.data.map(upstream => upstream.hue)), ); return { ...aux, mode: 'create' as const, record, - discovered: [], - modelsError: null, + discovered: null, preserveCredentials, }; } diff --git a/apps/web/src/routes/dashboard-providers-upstreams-edit.tsx b/apps/web/src/routes/dashboard-providers-upstreams-edit.tsx index fcc13af9e4..31d7b94fca 100644 --- a/apps/web/src/routes/dashboard-providers-upstreams-edit.tsx +++ b/apps/web/src/routes/dashboard-providers-upstreams-edit.tsx @@ -23,8 +23,7 @@ export async function clientLoader({ params }: Route.ClientLoaderArgs) { return { ...aux, record: recordResult.data, - discovered: [], - modelsError: null, + discovered: recordResult.data.cachedModels, mode: 'edit' as const, }; } diff --git a/apps/web/src/routes/dashboard-providers-upstreams-new.tsx b/apps/web/src/routes/dashboard-providers-upstreams-new.tsx index 03171809e6..f40d6202a3 100644 --- a/apps/web/src/routes/dashboard-providers-upstreams-new.tsx +++ b/apps/web/src/routes/dashboard-providers-upstreams-new.tsx @@ -21,12 +21,14 @@ export async function clientLoader({ params }: Route.ClientLoaderArgs) { if (!kind) { throw redirect('/dashboard/providers/upstreams'); } - const [recordResult, aux] = await Promise.all([ + const [recordResult, aux, upstreamsResult] = await Promise.all([ callApi(() => api.api.upstreams.blueprint.$get({ query: { kind } })), loadEditorAux(), + callApi(() => api.api.upstreams.$get()), ]); if (recordResult.error) throw new Error(recordResult.error.message); + if (upstreamsResult.error) throw new Error(upstreamsResult.error.message); const record = { ...recordResult.data, name: providerDefaultName[kind], @@ -34,9 +36,9 @@ export async function clientLoader({ params }: Route.ClientLoaderArgs) { // A blueprint carries no hue: the badge only has to be told apart from the // ones already on screen, which is a fact the dashboard holds and the // server does not. - hue: pickDistinctHue(aux.upstreams.map(upstream => upstream.hue)), + hue: pickDistinctHue(upstreamsResult.data.map(upstream => upstream.hue)), }; - return { ...aux, mode: 'create' as const, record, discovered: [], modelsError: null }; + return { ...aux, mode: 'create' as const, record, discovered: null }; } export const shouldRevalidate = revalidateOnPathnameChange; diff --git a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts index 2b7ab5d54f..59c00562ff 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts @@ -480,6 +480,46 @@ test('GET /api/upstreams attaches models-cache freshness to every row', async () }); }); +test('GET /api/upstreams/:id exposes cached editor rows without an upstream request', async () => { + const { repo, adminSession } = await setupAppTest(); + await repo.upstreams.deleteAll(); + const createdResponse = await requestApp('/api/upstreams', authed(adminSession, createBody({ + kind: 'copilot', + name: 'Cached Copilot', + config: copilotConfig, + disabled_public_model_ids: ['hidden-model'], + }))); + assertEquals(createdResponse.status, 201); + const created = (await createdResponse.json()) as { id: string }; + const read = () => requestApp(`/api/upstreams/${created.id}`, { headers: { 'x-floway-session': adminSession } }); + + const cold = await read(); + assertEquals(cold.status, 200); + assertEquals(((await cold.json()) as JsonObject).cachedModels, null); + + await seedModelsCache(repo.upstreams, created.id, await storedModelsRefreshIdentity(repo.upstreams, created.id), { + revision: MODEL_CATALOG_REVISION, + fetchedAt: 1_700_000_000_000, + models: [ + stubProviderModel({ id: 'visible-model', upstreamModelId: 'visible-model', display_name: 'Visible' }), + stubProviderModel({ id: 'hidden-model', upstreamModelId: 'hidden-model' }), + ], + }); + + await withMockedFetch( + request => { throw new Error(`Editor read unexpectedly fetched ${request.url}`); }, + async () => { + const response = await read(); + assertEquals(response.status, 200); + const body = (await response.json()) as JsonObject; + assertEquals(body.modelsCache.modelCount, 1); + assertEquals(body.cachedModels.map((model: { publicModelId: string }) => model.publicModelId), ['visible-model', 'hidden-model']); + assertEquals(body.cachedModels[0].display_name, 'Visible'); + assertEquals(body.cachedModels[0].providerData, undefined); + }, + ); +}); + test('GET /api/upstream-options returns the minimal picker shape to admin and non-admin callers', async () => { const { repo, adminSession, apiKey } = await setupAppTest(); await saveUpstreamForTest(repo.upstreams, { @@ -736,8 +776,59 @@ test('POST /api/upstreams/:id/list-models reads the saved config and publishes a assertEquals(upstreamCalls, 1); const cached = (await repo.upstreams.getById(savedRecord.id))?.modelsCache; assertEquals(cached?.models.map((model: { id: string }) => model.id), ['fresh-model']); + assertEquals(cached?.discovered?.map(model => model.upstreamModelId), ['fresh-model']); + const stored = await requestApp(`/api/upstreams/${savedRecord.id}`, { headers: { 'x-floway-session': adminSession } }); + assertEquals(stored.status, 200); + assertEquals(((await stored.json()) as JsonObject).cachedModels.map((model: { upstreamModelId: string }) => model.upstreamModelId), ['fresh-model']); + assertEquals(upstreamCalls, 1); + }, + ); +}); + +test('Custom cache retains discovered rows excluded from its routable catalog', async () => { + const { repo, adminSession } = await setupAppTest(); + await repo.upstreams.deleteAll(); + const response = await requestApp('/api/upstreams', authed(adminSession, createBody({ + config: { + ...customConfig, + modelsFetch: { enabled: true }, + models: [{ upstreamModelId: 'overridden', kind: 'chat', endpoints: { openaiChatCompletions: {} } }], + }, + }))); + assertEquals(response.status, 201); + const { id } = (await response.json()) as { id: string }; + + await seedModelsCache(repo.upstreams, id, await storedModelsRefreshIdentity(repo.upstreams, id), { + revision: MODEL_CATALOG_REVISION, + fetchedAt: 100, + models: [stubProviderModel({ id: 'overridden', upstreamModelId: 'overridden' })], + }); + const previous = await requestApp(`/api/upstreams/${id}`, { headers: { 'x-floway-session': adminSession } }); + assertEquals(((await previous.json()) as JsonObject).cachedModels, null); + + await withMockedFetch( + request => { + if (new URL(request.url).hostname !== 'custom.example.com') throw new Error(`Unexpected fetch ${request.url}`); + return jsonResponse({ + data: [ + { id: 'overridden' }, + { id: 'auto-only' }, + { id: 'rerank-only', kind: 'rerank' }, + ], + }); + }, + async () => { + const fetched = await requestApp(`/api/upstreams/${id}/list-models`, { method: 'POST', headers: { 'x-floway-session': adminSession } }); + assertEquals(fetched.status, 200); }, ); + + const cache = (await repo.upstreams.getById(id))?.modelsCache; + assertEquals(cache?.models.map(model => model.upstreamModelId), ['overridden', 'auto-only']); + assertEquals(cache?.discovered?.map(model => model.upstreamModelId), ['overridden', 'auto-only', 'rerank-only']); + const stored = await requestApp(`/api/upstreams/${id}`, { headers: { 'x-floway-session': adminSession } }); + assertEquals(stored.status, 200); + assertEquals(((await stored.json()) as JsonObject).cachedModels.map((model: { upstreamModelId: string }) => model.upstreamModelId), ['overridden', 'auto-only', 'rerank-only']); }); test('POST /api/upstreams/:id/list-models rejects a missing saved upstream', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts index 93c13541c2..43a30a64e6 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts @@ -559,7 +559,7 @@ test('OpenAI Responses WebSocket keep-alive waits for the first event and takes test('OpenAI Responses WebSocket returns OpenAI-style error envelopes for unsupported client events', async () => { const { apiKey } = await setupAppTest(); - await withWorkerWebSocketRuntime(async () => { + await withSuccessfulOpenAIResponsesUpstream(async () => await withWorkerWebSocketRuntime(async () => { const client = await connectOpenAIResponsesWebSocket(apiKey.key); const received = waitForMessages(client, messages => messages.length === 1); @@ -575,12 +575,12 @@ test('OpenAI Responses WebSocket returns OpenAI-style error envelopes for unsupp message: "Unsupported WebSocket event type 'session.update'.", }, }]); - }); + })); }); test('OpenAI Responses WebSocket returns invalid_request_error for malformed client messages', async () => { const { apiKey } = await setupAppTest(); - await withWorkerWebSocketRuntime(async () => { + await withSuccessfulOpenAIResponsesUpstream(async () => await withWorkerWebSocketRuntime(async () => { const client = await connectOpenAIResponsesWebSocket(apiKey.key); const invalidJson = waitForMessages(client, messages => messages.length === 1); @@ -656,7 +656,7 @@ test('OpenAI Responses WebSocket returns invalid_request_error for malformed cli param: 'input[0]', }, }]); - }); + })); }); test('OpenAI Responses WebSocket forwards HTTP failures with status, error.code, and event_id', async () => { diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index 6aff89563a..464696e5de 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -809,7 +809,7 @@ class MemoryUpstreamRepo implements UpstreamRepo { const existing = this.store.get(id); if (!existing || existing.configVersion !== configVersion || !matchesModelsRefreshInputs(existing, refreshInputs) || (existing.modelsCache?.fetchedAt ?? 0) !== cacheEpoch) return Promise.resolve(false); - existing.modelsCache = { revision: cache.revision, fetchedAt: cache.fetchedAt, models: [...cache.models], lastError: null }; + existing.modelsCache = { ...cache, models: [...cache.models], lastError: null }; return Promise.resolve(true); } @@ -831,7 +831,7 @@ const cloneUpstreamRecord = (upstream: StoredUpstreamRecord): StoredUpstreamReco ...upstream, config: structuredClone(upstream.config), state: upstream.state === null || upstream.state === undefined ? null : structuredClone(upstream.state), - modelsCache: upstream.modelsCache === null ? null : { ...upstream.modelsCache, models: [...upstream.modelsCache.models] }, + modelsCache: structuredClone(upstream.modelsCache), flagOverrides: normalizeFlagOverrides(upstream.flagOverrides), disabledPublicModelIds: normalizeDisabledPublicModelIds(upstream.disabledPublicModelIds), proxyFallbackList: normalizeProxyFallbackList(upstream.proxyFallbackList), diff --git a/packages/gateway/__tests__/repo/upstream-codecs_test.ts b/packages/gateway/__tests__/repo/upstream-codecs_test.ts new file mode 100644 index 0000000000..4142ff95d2 --- /dev/null +++ b/packages/gateway/__tests__/repo/upstream-codecs_test.ts @@ -0,0 +1,27 @@ +import { expect, test } from 'vitest'; + +import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; +import { decodeUpstreamModelsCache, encodeUpstreamModelsCache } from '../../src/repo/upstream-codecs.ts'; +import type { UpstreamModelsCache } from '@floway-dev/provider'; + +test('stored model cache preserves Custom editor discovery outside the routable catalog', () => { + const cache: UpstreamModelsCache = { + revision: MODEL_CATALOG_REVISION, + fetchedAt: 100, + models: [], + discovered: [{ + upstreamModelId: 'rerank-only', + publicModelId: 'rerank-only', + kind: 'rerank', + endpoints: { rerank: {} }, + opaqueBlobCompatibilityScope: { bindToUpstream: true }, + }], + lastError: null, + }; + expect(decodeUpstreamModelsCache(encodeUpstreamModelsCache(cache), 'up_custom')).toEqual(cache); +}); + +test('existing model caches without Custom discovery remain readable', () => { + const cache: UpstreamModelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 100, models: [], lastError: null }; + expect(decodeUpstreamModelsCache(encodeUpstreamModelsCache(cache), 'up_existing')).toEqual(cache); +}); diff --git a/packages/gateway/src/control-plane/upstreams/models-cache-projection.ts b/packages/gateway/src/control-plane/upstreams/models-cache-projection.ts new file mode 100644 index 0000000000..a99fe83821 --- /dev/null +++ b/packages/gateway/src/control-plane/upstreams/models-cache-projection.ts @@ -0,0 +1,33 @@ +import type { ListedUpstreamModel, ModelsCacheStatus } from './types.ts'; +import { storedCatalogSize } from '../../data-plane/providers/catalog.ts'; +import type { StoredUpstreamRecord } from '../../repo/types.ts'; +import type { ProviderModel, UpstreamModelConfig } from '@floway-dev/provider'; + +export const reshapeModelForDashboard = (model: ProviderModel): ListedUpstreamModel => ({ + upstreamModelId: model.upstreamModelId, + publicModelId: model.id, + kind: model.kind, + endpoints: model.endpoints, + ...(model.display_name !== undefined ? { display_name: model.display_name } : {}), + ...(Object.keys(model.limits).length > 0 ? { limits: model.limits } : {}), + ...(model.pricing ? { pricing: model.pricing } : {}), + ...(model.chat ? { chat: model.chat } : {}), + opaqueBlobCompatibilityScope: model.opaqueBlobCompatibilityScope, + ...(model.flagOverrides ? { flagOverrides: model.flagOverrides } : {}), +}); + +export const cachedModelsForDashboard = (record: StoredUpstreamRecord): UpstreamModelConfig[] | null => { + const cache = record.modelsCache; + if (cache === null || cache.fetchedAt <= 0) return null; + if (record.kind === 'custom') return cache.discovered ?? null; + if (record.kind === 'azure') return []; + return cache.models.map(reshapeModelForDashboard); +}; + +export const modelsCacheStatus = (record: StoredUpstreamRecord): ModelsCacheStatus => ({ + fetchedAt: record.modelsCache && record.modelsCache.fetchedAt > 0 ? record.modelsCache.fetchedAt : null, + lastError: record.modelsCache?.lastError + ? { message: record.modelsCache.lastError.message, at: record.modelsCache.lastError.at } + : null, + modelCount: record.modelsCache && record.modelsCache.fetchedAt > 0 ? storedCatalogSize(record) : null, +}); diff --git a/packages/gateway/src/control-plane/upstreams/models-cache-status.ts b/packages/gateway/src/control-plane/upstreams/models-cache-status.ts deleted file mode 100644 index a48196c2e6..0000000000 --- a/packages/gateway/src/control-plane/upstreams/models-cache-status.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { ModelsCacheStatus } from './types.ts'; -import { storedCatalogSize } from '../../data-plane/providers/catalog.ts'; -import type { StoredUpstreamRecord } from '../../repo/types.ts'; - -export const modelsCacheStatus = (record: StoredUpstreamRecord): ModelsCacheStatus => ({ - fetchedAt: record.modelsCache && record.modelsCache.fetchedAt > 0 ? record.modelsCache.fetchedAt : null, - lastError: record.modelsCache?.lastError - ? { message: record.modelsCache.lastError.message, at: record.modelsCache.lastError.at } - : null, - modelCount: record.modelsCache && record.modelsCache.fetchedAt > 0 ? storedCatalogSize(record) : null, -}); diff --git a/packages/gateway/src/control-plane/upstreams/models.ts b/packages/gateway/src/control-plane/upstreams/models.ts index 807d8a8c32..5005c2e69f 100644 --- a/packages/gateway/src/control-plane/upstreams/models.ts +++ b/packages/gateway/src/control-plane/upstreams/models.ts @@ -1,29 +1,15 @@ -import { modelsCacheStatus } from './models-cache-status.ts'; +import { modelsCacheStatus, reshapeModelForDashboard } from './models-cache-projection.ts'; import { upstreamErrorMessage as errorMessage } from './shared.ts'; -import type { ListedUpstreamModel } from './types.ts'; import { discoverDraftModels, isModelsRefreshConfigurationError, modelsRefreshErrorMessage, modelsRefreshTarget, refreshModelsExplicit } from '../../execution/models-refresh.ts'; import type { AuthedContext } from '../../middleware/auth.ts'; import type { CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { previewModelsBody } from '../schemas.ts'; -import { ProviderModelsUnavailableError, type ProviderModel, type UpstreamRecord } from '@floway-dev/provider'; +import { ProviderModelsUnavailableError, type UpstreamRecord } from '@floway-dev/provider'; const MODEL_LISTING_FAILURE_CODE = 'upstream_model_listing_failed'; -const reshapeModelForDashboard = (model: ProviderModel): ListedUpstreamModel => ({ - upstreamModelId: model.upstreamModelId, - publicModelId: model.id, - kind: model.kind, - endpoints: model.endpoints, - ...(model.display_name !== undefined ? { display_name: model.display_name } : {}), - ...(Object.keys(model.limits).length > 0 ? { limits: model.limits } : {}), - ...(model.pricing ? { pricing: model.pricing } : {}), - ...(model.chat ? { chat: model.chat } : {}), - opaqueBlobCompatibilityScope: model.opaqueBlobCompatibilityScope, - ...(model.flagOverrides ? { flagOverrides: model.flagOverrides } : {}), -}); - const malformedConfigResponse = (error: unknown): boolean => error instanceof Error && /Malformed .* upstream config/.test(error.message); diff --git a/packages/gateway/src/control-plane/upstreams/routes.ts b/packages/gateway/src/control-plane/upstreams/routes.ts index 9f89526dd9..40a70fcc4f 100644 --- a/packages/gateway/src/control-plane/upstreams/routes.ts +++ b/packages/gateway/src/control-plane/upstreams/routes.ts @@ -1,6 +1,6 @@ import type { Context } from 'hono'; -import { modelsCacheStatus } from './models-cache-status.ts'; +import { cachedModelsForDashboard, modelsCacheStatus } from './models-cache-projection.ts'; import { blueprintUpstreamRecord, upstreamRecordToFullJson, upstreamRecordToJson } from './serialize.ts'; import { isValidProviderKind, upstreamErrorMessage as errorMessage } from './shared.ts'; import type { FullSerializedUpstreamRecord, ModelsCacheStatus, RedactedSerializedUpstreamRecord } from './types.ts'; @@ -203,15 +203,17 @@ export const getUpstreamBlueprint = (c: Context) => { // Single-record read for the edit page. Returns the FULL record — no // secret redaction — because every editor-scoped action posts the record // back to a helper endpoint that needs the same credentials the data plane -// uses (refresh tokens, api keys, etc.). Codex quota and modelsCache are -// response-only projections, so they are attached here alongside the -// unredacted config/state — the edit page relies on `modelsCache` to -// render the "last fetched / last error" panel on mount. +// uses (refresh tokens, api keys, etc.). The cache status and editor model +// snapshot are read from this same row, so opening the editor displays the +// stored catalog without starting or waiting for a refresh. export const getUpstream = async (c: AuthedContext<'/:id'>) => { const id = c.req.param('id'); const [record, knownProxyIds] = await Promise.all([getRepo().upstreams.getById(id), loadKnownProxyIds()]); if (!record) return c.json({ error: 'upstream not found' }, 404); - return c.json(await serializeForResponse(record, knownProxyIds, upstreamRecordToFullJson)); + return c.json({ + ...await serializeForResponse(record, knownProxyIds, upstreamRecordToFullJson), + cachedModels: cachedModelsForDashboard(record), + }); }; export const createUpstream = async (c: CtxWithJson) => { diff --git a/packages/gateway/src/execution/models-refresh.ts b/packages/gateway/src/execution/models-refresh.ts index 5050cb8946..bb46c958f8 100644 --- a/packages/gateway/src/execution/models-refresh.ts +++ b/packages/gateway/src/execution/models-refresh.ts @@ -148,7 +148,7 @@ const discoverModels = async (record: UpstreamRecord, fetcher: Fetcher, fetchCus }> => await withRedactedCredentialEcho(fetcher, async trackedFetcher => { if (record.kind === 'custom') { const custom = assertCustomUpstreamRecord(record); - if (!fetchCustomLive && !custom.config.modelsFetch.enabled) return { models: projectCustomModels(record) }; + if (!fetchCustomLive && !custom.config.modelsFetch.enabled) return { models: projectCustomModels(record), discovered: [] }; const response = await fetchCustomModels(custom.config, trackedFetcher); return { models: projectCustomModels(record, response), discovered: projectCustomDiscoveredModels(record, response) }; } @@ -197,7 +197,12 @@ export const executeModelsRefresh = async (input: ModelsRefreshExecutionInput): configVersion: input.configVersion, cacheEpoch: epoch, refreshInputs: modelsRefreshInputs(record), - cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: Math.max(Date.now(), epoch + 1), models: result.models }, + cache: { + revision: MODEL_CATALOG_REVISION, + fetchedAt: Math.max(Date.now(), epoch + 1), + models: result.models, + ...(record.kind === 'custom' ? { discovered: result.discovered! } : {}), + }, }); return { kind: 'discovered', ...result, publication: published ? 'published' : 'lost-race' }; }; diff --git a/packages/gateway/src/repo/upstream-codecs.ts b/packages/gateway/src/repo/upstream-codecs.ts index d05c558dc6..6418c39be3 100644 --- a/packages/gateway/src/repo/upstream-codecs.ts +++ b/packages/gateway/src/repo/upstream-codecs.ts @@ -93,10 +93,28 @@ const providerModelSchema = z.object({ flagOverrides: flagOverridesSchema.optional(), }).passthrough(); +const discoveredModelSchema = z.object({ + upstreamModelId: z.string(), + publicModelId: z.string().optional(), + display_name: z.string().optional(), + limits: limitsSchema.optional(), + kind: z.enum(MODEL_KINDS), + pricing: pricingSchema.optional(), + chat: chatSchema.optional(), + endpoints: endpointsSchema, + opaqueBlobCompatibilityScope: opaqueBlobCompatibilityScopeSchema.optional(), + rerankTarget: z.object({ + protocol: z.enum(RERANK_PROTOCOLS), + path: z.string().optional(), + }).passthrough().optional(), + flagOverrides: flagOverridesSchema.optional(), +}).passthrough(); + const modelsCacheSchema = z.object({ revision: z.number(), fetchedAt: z.number(), models: z.array(providerModelSchema), + discovered: z.array(discoveredModelSchema).optional(), lastError: z.object({ message: z.string(), at: z.number(), failureCount: z.number().int().positive() }).passthrough().nullable(), }).passthrough(); const modelsCacheEnvelopeSchema = z.object({ revision: z.number() }).passthrough(); diff --git a/packages/provider/src/model.ts b/packages/provider/src/model.ts index 0a85b0c2c4..6cbb3ee011 100644 --- a/packages/provider/src/model.ts +++ b/packages/provider/src/model.ts @@ -1,5 +1,5 @@ import type { FlagId, FlagOverrides } from './flags.ts'; -import type { UpstreamChatModelConfig } from './model-config.ts'; +import type { UpstreamChatModelConfig, UpstreamModelConfig } from './model-config.ts'; import type { ModelPrefixConfig } from './model-prefix.ts'; import type { AliasSelection, AliasTarget, ModelKind, ModelEndpoints, ModelPricing, OpaqueBlobCompatibilityScope, PublicModelLimits, RerankTarget } from '@floway-dev/protocols/common'; @@ -55,6 +55,9 @@ export interface UpstreamModelsCache { revision: number; fetchedAt: number; models: ProviderModel[]; + // Custom's editable auto rows include models overridden by manual entries + // and rerank rows that are not part of the routable provider catalog. + discovered?: UpstreamModelConfig[]; lastError: { message: string; at: number; failureCount: number } | null; }