PQA: bring in line with llm_config changes in LDP AAI-440

pyproject.toml

@@ -30,7 +30,7 @@ classifiers = [
 dependencies = [
     "anyio",
     "fhaviary[llm]>=0.34",  # For type(action)=Message
-    "fhlmi>=0.45.0",  # For type(action)=Message
+    "fhlmi~=1.0",  # For LLMConfig-based model configuration
     "html2text",  # TODO: evaluate moving to an opt-in dependency
     "httpx",
     "httpx-aiohttp",
@@ -60,7 +60,6 @@ requires-python = ">=3.11"
 
 [project.optional-dependencies]
 dev = [
-    "fhlmi>=0.44.0",  # Lower pin for update from Haiku 3.5 to 4.5
     "httpx-aiohttp>=0.1.11",  # Pin for raw headers fix
     "ipykernel>=6.29",  # For running Jupter notebooks, and pin to keep recent
     "ipython>=8",  # Pin to keep recent
@@ -89,7 +88,7 @@ dev = [
 docling = ["paper-qa-docling"]
 image = ["fhlmi[image]"]
 ldp = [
-    "ldp>=0.45.0,<1",  # Lower pin for type(action)=Message, upper pin if v1 introduces breaks
+    "ldp~=1.0",  # For LLMConfig-based agents
 ]
 local = [
     "sentence-transformers",

src/paperqa/agents/helpers.py

@@ -55,10 +55,7 @@ async def litellm_get_search_query(
         )
 
     if isinstance(llm, str):
-        model: LLMModel = LiteLLMModel(name=llm)
-        model.config["model_list"][0]["litellm_params"].update(
-            {"temperature": temperature}
-        )
+        model: LLMModel = LiteLLMModel(name=llm, config={"temperature": temperature})
     else:
         model = llm
     messages = [

src/paperqa/agents/main.py

@@ -9,7 +9,6 @@
     Tool,
     ToolCall,
     ToolRequestMessage,
-    ToolSelector,
     ToolSelectorLedger,
 )
 from aviary.utils import MultipleChoiceQuestion
@@ -44,6 +43,7 @@
     from aviary.core import Environment
     from ldp.agent import Agent, SimpleAgentState
     from ldp.graph.ops import OpResult
+    from lmi import LiteLLMModel
 
 logger = logging.getLogger(__name__)
 agent_logger = logging.getLogger(__name__ + ".agent_callers")
@@ -127,9 +127,9 @@ async def run_agent(
         session, agent_status = await run_fake_agent(
             query, settings, docs, **runner_kwargs
         )
-    elif tool_selector_or_none := settings.make_aviary_tool_selector(agent_type):
+    elif tool_selector_model := settings.get_tool_selector_model(agent_type):
         session, agent_status = await run_aviary_agent(
-            query, settings, docs, tool_selector_or_none, **runner_kwargs
+            query, settings, docs, tool_selector_model, **runner_kwargs
         )
     elif ldp_agent_or_none := await settings.make_ldp_agent(agent_type):
         session, agent_status = await run_ldp_agent(
@@ -260,7 +260,7 @@ async def run_aviary_agent(
     query: str | MultipleChoiceQuestion,
     settings: Settings,
     docs: Docs,
-    agent: ToolSelector,
+    llm_model: "LiteLLMModel",
     env_class: type[PaperQAEnvironment] = PaperQAEnvironment,
     on_env_reset_callback: Callable[[EnvironmentState], Awaitable] | None = None,
     on_agent_action_callback: Callable[[Message, BaseModel], Awaitable] | None = None,
@@ -308,8 +308,8 @@ async def rollout() -> AgentStatus:
                 before_sleep=before_sleep_log(logger, logging.WARNING),
                 reraise=True,
             ):
-                with attempt:  # Retrying if ToolSelector fails to select a tool
-                    action = await agent(agent_state.messages, tools)
+                with attempt:  # Retry malformed tool selections
+                    action = await llm_model.select_tool(agent_state.messages, tools)
             agent_state.messages = [*agent_state.messages, action]
             if on_agent_action_callback:
                 await on_agent_action_callback(action, agent_state)

src/paperqa/configs/debug.json

@@ -1,6 +1,6 @@
 {
-  "llm": "claude-3-haiku-20240307",
-  "summary_llm": "claude-3-haiku-20240307",
+  "llm": "claude-haiku-4-5-20251001",
+  "summary_llm": "claude-haiku-4-5-20251001",
   "answer": {
     "evidence_k": 2,
     "evidence_summary_length": "25 to 50 words",

src/paperqa/contrib/openreview_paper_helper.py

@@ -8,7 +8,6 @@
 import httpx
 import httpx_aiohttp
 from aviary.core import Message
-from lmi import LiteLLMModel
 from pydantic import BaseModel, Field
 
 from paperqa import Docs, Settings
@@ -59,9 +58,7 @@ def __init__(
             password=password or os.getenv("OPENREVIEW_PASSWORD"),
         )
         self.venue_id = venue_id
-        self.llm_model = LiteLLMModel(
-            name=self.settings.llm, config=self.settings.llm_config
-        )
+        self.llm_model = settings.get_llm()
 
     def get_venues(self) -> list[str]:
         """Get list of available venues."""

src/paperqa/core.py

@@ -7,6 +7,7 @@
 import litellm
 from aviary.core import Message
 from lmi import LLMModel, LLMResult
+from lmi.exceptions import AllModelsExhaustedError
 from pydantic import JsonValue
 
 from paperqa.prompts import text_with_tables_prompt_template
@@ -164,7 +165,8 @@ class LLMContextRequestFailedError(LLMContextError):
     """Non-retryable exception for when the LLM provider fails to respond.
 
     Kind of a catch-all for intermittent failures, safety refusals, etc.
-    Catches all litellm.BadRequestErrors and litellm.MidStreamFallbackErrors.
+    Catches litellm.BadRequestErrors, bare or wrapped in an
+    lmi.exceptions.AllModelsExhaustedError; other exhaustion causes propagate.
     """
 
     retryable = False
@@ -267,7 +269,13 @@ async def _map_fxn_summary(  # noqa: PLR0912
                     callbacks=callbacks,
                     name="evidence:" + text.name,
                 )
-            except litellm.BadRequestError as exc:
+            except (litellm.BadRequestError, AllModelsExhaustedError) as exc:
+                # Provider-specific 400s (e.g. Anthropic's 100-image limit)
+                # arrive wrapped in AllModelsExhaustedError
+                if isinstance(exc, AllModelsExhaustedError) and not isinstance(
+                    exc.last_exc, litellm.BadRequestError
+                ):
+                    raise
                 if not evidence_text_only_fallback:
                     raise
                 logger.warning(
@@ -286,17 +294,22 @@ async def _map_fxn_summary(  # noqa: PLR0912
                     name="evidence:" + text.name,
                 )
                 used_text_only_fallback = True
-        except litellm.Timeout as exc:
-            raise LLMContextTimeoutError(
-                f"LLM call to create a context timed out on text named {text.name!r}.",
+        except (litellm.Timeout, AllModelsExhaustedError) as exc:
+            root_exc = exc.last_exc if isinstance(exc, AllModelsExhaustedError) else exc
+            if isinstance(root_exc, litellm.Timeout):
+                raise LLMContextTimeoutError(
+                    f"LLM call to create a context timed out"
+                    f" on text named {text.name!r}.",
+                    llm_results=llm_results,
+                ) from exc
+            if not isinstance(root_exc, litellm.BadRequestError):
+                # Rate limits, 5xx, auth: propagate rather than drop the context
+                raise
+            raise LLMContextRequestFailedError(
+                f"LLM call to create a context failed on text named {text.name!r}.",
                 llm_results=llm_results,
             ) from exc
-        except (
-            litellm.exceptions.MidStreamFallbackError,
-            litellm.BadRequestError,
-        ) as exc:
-            # BadRequestError: what is thrown if you directly call an LLM with a bad request
-            # MidStreamFallbackError: what litellm throws if there are fallbacks configured
+        except litellm.BadRequestError as exc:
             raise LLMContextRequestFailedError(
                 f"LLM call to create a context failed on text named {text.name!r}.",
                 llm_results=llm_results,