From 868033c8ce6e92ae70c801364fd7a53d4c4d563e Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 21 Apr 2026 08:36:34 +0200 Subject: [PATCH 1/5] ref(langgraph): Revert input truncation --- sentry_sdk/integrations/langgraph.py | 31 ++++------ .../integrations/langgraph/test_langgraph.py | 58 ------------------- 2 files changed, 10 insertions(+), 79 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index e5ea12b90a..bd1c685057 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -5,7 +5,6 @@ from sentry_sdk.ai.utils import ( set_data_normalized, normalize_message_roles, - truncate_and_annotate_messages, ) from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -181,17 +180,12 @@ def new_invoke(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": input_messages = _parse_langgraph_messages(args[0]) if input_messages: normalized_input_messages = normalize_message_roles(input_messages) - scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_input_messages, span, scope + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_input_messages, + unpack=False, ) - if messages_data is not None: - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - messages_data, - unpack=False, - ) result = f(self, *args, **kwargs) @@ -234,17 +228,12 @@ async def new_ainvoke(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": input_messages = _parse_langgraph_messages(args[0]) if input_messages: normalized_input_messages = normalize_message_roles(input_messages) - scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_input_messages, span, scope + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_input_messages, + unpack=False, ) - if messages_data is not None: - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - messages_data, - unpack=False, - ) result = await f(self, *args, **kwargs) diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index e1a3baa0a8..91fea2d760 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -1351,61 +1351,3 @@ def __init__(self, content, message_type="human"): # Verify no "ai" roles remain roles = [msg["role"] for msg in stored_messages if "role" in msg] assert "ai" not in roles - - -def test_langgraph_message_truncation(sentry_init, capture_items): - """Test that large messages are truncated properly in Langgraph integration.""" - import json - - sentry_init( - integrations=[LanggraphIntegration(include_prompts=True)], - traces_sample_rate=1.0, - send_default_pii=True, - ) - items = capture_items("transaction", "span") - - large_content = ( - "This is a very long message that will exceed our size limits. " * 1000 - ) - test_state = { - "messages": [ - MockMessage("small message 1", name="user"), - MockMessage(large_content, name="assistant"), - MockMessage(large_content, name="user"), - MockMessage("small message 4", name="assistant"), - MockMessage("small message 5", name="user"), - ] - } - - pregel = MockPregelInstance("test_graph") - - def original_invoke(self, *args, **kwargs): - return {"messages": args[0].get("messages", [])} - - with start_transaction(): - wrapped_invoke = _wrap_pregel_invoke(original_invoke) - result = wrapped_invoke(pregel, test_state) - - assert result is not None - - spans = [item.payload for item in items if item.type == "span"] - invoke_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == OP.GEN_AI_INVOKE_AGENT - ] - assert len(invoke_spans) > 0 - - invoke_span = invoke_spans[0] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["attributes"] - - messages_data = invoke_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) - - (tx,) = (item.payload for item in items if item.type == "transaction") - assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 From 08059f5a0ad00712b614e9ddd092b57a8e711dcd Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 21 Apr 2026 08:41:19 +0200 Subject: [PATCH 2/5] ref(langchain): Revert input truncation --- sentry_sdk/integrations/langchain.py | 61 +++++---------- .../integrations/langchain/test_langchain.py | 78 ------------------- 2 files changed, 20 insertions(+), 119 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 52a7fe6695..438137becf 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -14,7 +14,6 @@ get_start_span_function, normalize_message_roles, set_data_normalized, - truncate_and_annotate_messages, ) from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -377,17 +376,12 @@ def on_llm_start( } for prompt in prompts ] - scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) - if messages_data is not None: - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - messages_data, - unpack=False, - ) def on_chat_model_start( self: "SentryLangchainCallback", @@ -457,17 +451,12 @@ def on_chat_model_start( self._normalize_langchain_message(message) ) normalized_messages = normalize_message_roles(normalized_messages) - scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) - if messages_data is not None: - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - messages_data, - unpack=False, - ) def on_chat_model_end( self: "SentryLangchainCallback", @@ -979,17 +968,12 @@ def new_invoke(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": and integration.include_prompts ): normalized_messages = normalize_message_roles([input]) - scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) - if messages_data is not None: - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - messages_data, - unpack=False, - ) output = result.get("output") if ( @@ -1041,17 +1025,12 @@ def new_stream(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": and integration.include_prompts ): normalized_messages = normalize_message_roles([input]) - scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) - if messages_data is not None: - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - messages_data, - unpack=False, - ) # Run the agent result = f(self, *args, **kwargs) diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 319b96a06a..2fd7b953e8 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -1290,84 +1290,6 @@ def test_langchain_message_role_normalization_units(): assert normalized[5] == "string message" # String message unchanged -def test_langchain_message_truncation(sentry_init, capture_items): - """Test that large messages are truncated properly in Langchain integration.""" - from langchain_core.outputs import LLMResult, Generation - - sentry_init( - integrations=[LangchainIntegration(include_prompts=True)], - traces_sample_rate=1.0, - send_default_pii=True, - ) - items = capture_items("transaction", "span") - - callback = SentryLangchainCallback(max_span_map_size=100, include_prompts=True) - - run_id = "12345678-1234-1234-1234-123456789012" - serialized = {"_type": "openai-chat", "model_name": "gpt-3.5-turbo"} - - large_content = ( - "This is a very long message that will exceed our size limits. " * 1000 - ) - prompts = [ - "small message 1", - large_content, - large_content, - "small message 4", - "small message 5", - ] - - with start_transaction(): - callback.on_llm_start( - serialized=serialized, - prompts=prompts, - run_id=run_id, - name="my_pipeline", - invocation_params={ - "temperature": 0.7, - "max_tokens": 100, - "model": "gpt-3.5-turbo", - }, - ) - - response = LLMResult( - generations=[[Generation(text="The response")]], - llm_output={ - "token_usage": { - "total_tokens": 25, - "prompt_tokens": 10, - "completion_tokens": 15, - } - }, - ) - callback.on_llm_end(response=response, run_id=run_id) - - tx = next(item.payload for item in items if item.type == "transaction") - assert tx["type"] == "transaction" - - spans = [item.payload for item in items if item.type == "span"] - llm_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.text_completion" - ] - assert len(llm_spans) > 0 - - llm_span = llm_spans[0] - assert llm_span["attributes"]["gen_ai.operation.name"] == "text_completion" - assert llm_span["attributes"][SPANDATA.GEN_AI_PIPELINE_NAME] == "my_pipeline" - - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in llm_span["attributes"] - messages_data = llm_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) - assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 - - @pytest.mark.parametrize( "send_default_pii, include_prompts", [ From abcda60aa02c11cf6926dc8e5b78f9be0d7cac45 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 21 Apr 2026 08:43:30 +0200 Subject: [PATCH 3/5] revert langchain changes --- sentry_sdk/integrations/langchain.py | 99 +- .../integrations/langchain/test_langchain.py | 983 ++++++++++++------ 2 files changed, 748 insertions(+), 334 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 438137becf..49fa04c034 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -14,6 +14,8 @@ get_start_span_function, normalize_message_roles, set_data_normalized, + truncate_and_annotate_messages, + transform_content_part, ) from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -127,6 +129,39 @@ def _get_ai_system(all_params: "Dict[str, Any]") -> "Optional[str]": } +def _transform_langchain_content_block( + content_block: "Dict[str, Any]", +) -> "Dict[str, Any]": + """ + Transform a LangChain content block using the shared transform_content_part function. + + Returns the original content block if transformation is not applicable + (e.g., for text blocks or unrecognized formats). + """ + result = transform_content_part(content_block) + return result if result is not None else content_block + + +def _transform_langchain_message_content(content: "Any") -> "Any": + """ + Transform LangChain message content, handling both string content and + list of content blocks. + """ + if isinstance(content, str): + return content + + if isinstance(content, (list, tuple)): + transformed = [] + for block in content: + if isinstance(block, dict): + transformed.append(_transform_langchain_content_block(block)) + else: + transformed.append(block) + return transformed + + return content + + # Contextvar to track agent names in a stack for re-entrant agent support _agent_stack: "contextvars.ContextVar[Optional[List[Optional[str]]]]" = ( contextvars.ContextVar("langchain_agent_stack", default=None) @@ -278,7 +313,9 @@ def _handle_error(self, run_id: "UUID", error: "Any") -> None: del self.span_map[run_id] def _normalize_langchain_message(self, message: "BaseMessage") -> "Any": - parsed = {"role": message.type, "content": message.content} + # Transform content to handle multimodal data (images, audio, video, files) + transformed_content = _transform_langchain_message_content(message.content) + parsed = {"role": message.type, "content": transformed_content} parsed.update(message.additional_kwargs) return parsed @@ -376,12 +413,17 @@ def on_llm_start( } for prompt in prompts ] - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - normalized_messages, - unpack=False, + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages( + normalized_messages, span, scope ) + if messages_data is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + messages_data, + unpack=False, + ) def on_chat_model_start( self: "SentryLangchainCallback", @@ -451,12 +493,17 @@ def on_chat_model_start( self._normalize_langchain_message(message) ) normalized_messages = normalize_message_roles(normalized_messages) - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - normalized_messages, - unpack=False, + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages( + normalized_messages, span, scope ) + if messages_data is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + messages_data, + unpack=False, + ) def on_chat_model_end( self: "SentryLangchainCallback", @@ -968,12 +1015,17 @@ def new_invoke(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": and integration.include_prompts ): normalized_messages = normalize_message_roles([input]) - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - normalized_messages, - unpack=False, + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages( + normalized_messages, span, scope ) + if messages_data is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + messages_data, + unpack=False, + ) output = result.get("output") if ( @@ -1025,12 +1077,17 @@ def new_stream(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": and integration.include_prompts ): normalized_messages = normalize_message_roles([input]) - set_data_normalized( - span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - normalized_messages, - unpack=False, + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages( + normalized_messages, span, scope ) + if messages_data is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + messages_data, + unpack=False, + ) # Run the agent result = f(self, *args, **kwargs) diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 2fd7b953e8..498a5d6f4a 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -27,6 +27,8 @@ from sentry_sdk.integrations.langchain import ( LangchainIntegration, SentryLangchainCallback, + _transform_langchain_content_block, + _transform_langchain_message_content, ) try: @@ -95,7 +97,7 @@ def _llm_type(self) -> str: def test_langchain_text_completion( sentry_init, - capture_items, + capture_events, get_model_response, ): sentry_init( @@ -107,7 +109,7 @@ def test_langchain_text_completion( traces_sample_rate=1.0, send_default_pii=True, ) - items = capture_items("transaction", "span") + events = capture_events() model_response = get_model_response( Completion( @@ -147,29 +149,25 @@ def test_langchain_text_completion( input_text = "What is the capital of France?" model.invoke(input_text, config={"run_name": "my-snazzy-pipeline"}) - tx = next(item.payload for item in items if item.type == "transaction") + tx = events[0] assert tx["type"] == "transaction" - spans = [item.payload for item in items if item.type == "span"] llm_spans = [ span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.text_completion" + for span in tx.get("spans", []) + if span.get("op") == "gen_ai.text_completion" ] assert len(llm_spans) > 0 llm_span = llm_spans[0] - assert llm_span["name"] == "text_completion gpt-3.5-turbo" - assert llm_span["attributes"]["gen_ai.system"] == "openai" - assert llm_span["attributes"]["gen_ai.pipeline.name"] == "my-snazzy-pipeline" - assert llm_span["attributes"]["gen_ai.request.model"] == "gpt-3.5-turbo" - assert ( - llm_span["attributes"]["gen_ai.response.text"] - == "The capital of France is Paris." - ) - assert llm_span["attributes"]["gen_ai.usage.total_tokens"] == 25 - assert llm_span["attributes"]["gen_ai.usage.input_tokens"] == 10 - assert llm_span["attributes"]["gen_ai.usage.output_tokens"] == 15 + assert llm_span["description"] == "text_completion gpt-3.5-turbo" + assert llm_span["data"]["gen_ai.system"] == "openai" + assert llm_span["data"]["gen_ai.pipeline.name"] == "my-snazzy-pipeline" + assert llm_span["data"]["gen_ai.request.model"] == "gpt-3.5-turbo" + assert llm_span["data"]["gen_ai.response.text"] == "The capital of France is Paris." + assert llm_span["data"]["gen_ai.usage.total_tokens"] == 25 + assert llm_span["data"]["gen_ai.usage.input_tokens"] == 10 + assert llm_span["data"]["gen_ai.usage.output_tokens"] == 15 @pytest.mark.skipif( @@ -198,7 +196,7 @@ def test_langchain_text_completion( ) def test_langchain_create_agent( sentry_init, - capture_items, + capture_events, send_default_pii, include_prompts, system_instructions_content, @@ -215,7 +213,7 @@ def test_langchain_create_agent( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - items = capture_items("transaction", "span") + events = capture_events() model_response = get_model_response( nonstreaming_responses_model_response, @@ -252,23 +250,22 @@ def test_langchain_create_agent( }, ) - tx = next(item.payload for item in items if item.type == "transaction") + tx = events[0] assert tx["type"] == "transaction" assert tx["contexts"]["trace"]["origin"] == "manual" - spans = [item.payload for item in items if item.type == "span"] - chat_spans = list(x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat") + chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") assert len(chat_spans) == 1 - assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[0]["origin"] == "auto.ai.langchain" - assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" - assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 10 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 20 - assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30 + assert chat_spans[0]["data"]["gen_ai.system"] == "openai-chat" + assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 10 + assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 20 + assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 30 if send_default_pii and include_prompts: assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hello, how can I help you?" ) @@ -279,9 +276,7 @@ def test_langchain_create_agent( "type": "text", "content": "You are very powerful assistant, but don't know current events", } - ] == json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) + ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) else: assert [ { @@ -292,17 +287,11 @@ def test_langchain_create_agent( "type": "text", "content": "Be concise and clear.", }, - ] == json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) + ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get("data", {}) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {}) @pytest.mark.skipif( @@ -320,7 +309,7 @@ def test_langchain_create_agent( ) def test_tool_execution_span( sentry_init, - capture_items, + capture_events, send_default_pii, include_prompts, get_model_response, @@ -335,7 +324,7 @@ def test_tool_execution_span( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - items = capture_items("transaction", "span") + events = capture_events() responses = responses_tool_call_model_responses( tool_name="get_word_length", @@ -411,71 +400,60 @@ def test_tool_execution_span( }, ) - tx = next(item.payload for item in items if item.type == "transaction") + tx = events[0] assert tx["type"] == "transaction" assert tx["contexts"]["trace"]["origin"] == "manual" - spans = [item.payload for item in items if item.type == "span"] - chat_spans = list(x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat") + chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") assert len(chat_spans) == 2 - tool_exec_spans = list( - x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.execute_tool" - ) + tool_exec_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool") assert len(tool_exec_spans) == 1 tool_exec_span = tool_exec_spans[0] - assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[0]["origin"] == "auto.ai.langchain" + assert chat_spans[1]["origin"] == "auto.ai.langchain" + assert tool_exec_span["origin"] == "auto.ai.langchain" - assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" + assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142 + assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50 + assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192 + assert chat_spans[0]["data"]["gen_ai.system"] == "openai-chat" - assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 - assert chat_spans[1]["attributes"]["gen_ai.system"] == "openai-chat" + assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89 + assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28 + assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117 + assert chat_spans[1]["data"]["gen_ai.system"] == "openai-chat" if send_default_pii and include_prompts: - assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] + assert "word" in tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_INPUT] - assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "5" in chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get( - "attributes", {} - ), ( + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("data", {}), ( "Tool calls should be recorded when send_default_pii=True and include_prompts=True" ) - tool_calls_data = chat_spans[0]["attributes"][ - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS - ] + tool_calls_data = chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] assert isinstance(tool_calls_data, str) assert "get_word_length" in tool_calls_data else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) - assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {}) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("data", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("data", {}) + assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("data", {}) + assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("data", {}) # Verify tool calls are NOT recorded when PII is disabled assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( - "attributes", {} + "data", {} ), ( f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " f"and include_prompts={include_prompts}" ) assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( - "attributes", {} + "data", {} ), ( f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " f"and include_prompts={include_prompts}" @@ -483,7 +461,7 @@ def test_tool_execution_span( # Verify that available tools are always recorded regardless of PII settings for chat_span in chat_spans: - tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] + tools_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] assert "get_word_length" in tools_data @@ -510,7 +488,7 @@ def test_tool_execution_span( ) def test_langchain_openai_tools_agent( sentry_init, - capture_items, + capture_events, send_default_pii, include_prompts, system_instructions_content, @@ -527,7 +505,7 @@ def test_langchain_openai_tools_agent( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - items = capture_items("transaction", "span") + events = capture_events() prompt = ChatPromptTemplate.from_messages( [ @@ -722,47 +700,40 @@ def test_langchain_openai_tools_agent( with start_transaction(): list(agent_executor.stream({"input": "How many letters in the word eudca"})) - tx = next(item.payload for item in items if item.type == "transaction") + tx = events[0] assert tx["type"] == "transaction" assert tx["contexts"]["trace"]["origin"] == "manual" - spans = [item.payload for item in items if item.type == "span"] - invoke_agent_span = next( - x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.invoke_agent" - ) - chat_spans = list(x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat") - tool_exec_span = next( - x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.execute_tool" - ) + invoke_agent_span = next(x for x in tx["spans"] if x["op"] == "gen_ai.invoke_agent") + chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") + tool_exec_span = next(x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool") assert len(chat_spans) == 2 - assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert invoke_agent_span["origin"] == "auto.ai.langchain" + assert chat_spans[0]["origin"] == "auto.ai.langchain" + assert chat_spans[1]["origin"] == "auto.ai.langchain" + assert tool_exec_span["origin"] == "auto.ai.langchain" # We can't guarantee anything about the "shape" of the langchain execution graph - assert ( - len(list(x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat")) > 0 - ) + assert len(list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")) > 0 # Token usage is only available in newer versions of langchain (v0.2+) # where usage_metadata is supported on AIMessageChunk - if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: - assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 + if "gen_ai.usage.input_tokens" in chat_spans[0]["data"]: + assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142 + assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50 + assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192 - if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: - assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 + if "gen_ai.usage.input_tokens" in chat_spans[1]["data"]: + assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89 + assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28 + assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117 if send_default_pii and include_prompts: - assert "5" in chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] - assert 5 == int(tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_OUTPUT]) + assert "5" in chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "word" in tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_INPUT] + assert 5 == int(tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_OUTPUT]) param_id = request.node.callspec.id if "string" in param_id: @@ -771,9 +742,7 @@ def test_langchain_openai_tools_agent( "type": "text", "content": "You are very powerful assistant, but don't know current events", } - ] == json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) + ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) else: assert [ { @@ -784,21 +753,15 @@ def test_langchain_openai_tools_agent( "type": "text", "content": "Be concise and clear.", }, - ] == json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) + ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) - assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "5" in chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get( - "attributes", {} - ), ( + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("data", {}), ( "Tool calls should be recorded when send_default_pii=True and include_prompts=True" ) - tool_calls_data = chat_spans[0]["attributes"][ - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS - ] + tool_calls_data = chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] assert isinstance(tool_calls_data, (list, str)) # Could be serialized if isinstance(tool_calls_data, str): assert "get_word_length" in tool_calls_data @@ -807,55 +770,45 @@ def test_langchain_openai_tools_agent( tool_call_str = str(tool_calls_data) assert "get_word_length" in tool_call_str else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) - assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get("data", {}) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {}) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get("data", {}) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("data", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("data", {}) + assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("data", {}) + assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("data", {}) # Verify tool calls are NOT recorded when PII is disabled assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( - "attributes", {} + "data", {} ), ( f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " f"and include_prompts={include_prompts}" ) assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( - "attributes", {} + "data", {} ), ( f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " f"and include_prompts={include_prompts}" ) # Verify finish_reasons is always an array of strings - assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ "function_call" ] - assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "stop" - ] + assert chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["stop"] # Verify that available tools are always recorded regardless of PII settings for chat_span in chat_spans: - tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] + tools_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] assert tools_data is not None, ( "Available tools should always be recorded regardless of PII settings" ) assert "get_word_length" in tools_data -def test_langchain_error(sentry_init, capture_items): +def test_langchain_error(sentry_init, capture_events): global llm_type llm_type = "acme-llm" @@ -864,7 +817,7 @@ def test_langchain_error(sentry_init, capture_items): traces_sample_rate=1.0, send_default_pii=True, ) - items = capture_items("event", "transaction", "span") + events = capture_events() prompt = ChatPromptTemplate.from_messages( [ @@ -890,11 +843,11 @@ def test_langchain_error(sentry_init, capture_items): with start_transaction(), pytest.raises(ValueError): list(agent_executor.stream({"input": "How many letters in the word eudca"})) - (error,) = (item.payload for item in items if item.type == "event") + error = events[0] assert error["level"] == "error" -def test_span_status_error(sentry_init, capture_items): +def test_span_status_error(sentry_init, capture_events): global llm_type llm_type = "acme-llm" @@ -902,7 +855,7 @@ def test_span_status_error(sentry_init, capture_items): integrations=[LangchainIntegration(include_prompts=True)], traces_sample_rate=1.0, ) - items = capture_items("event", "transaction", "span") + events = capture_events() with start_transaction(name="test"): prompt = ChatPromptTemplate.from_messages( @@ -931,13 +884,10 @@ def test_span_status_error(sentry_init, capture_items): with pytest.raises(ValueError): list(agent_executor.stream({"input": "How many letters in the word eudca"})) - (error,) = (item.payload for item in items if item.type == "event") + (error, transaction) = events assert error["level"] == "error" - - spans = [item.payload for item in items if item.type == "span"] - assert spans[0]["status"] == "error" - - (transaction,) = (item.payload for item in items if item.type == "transaction") + assert transaction["spans"][0]["status"] == "internal_error" + assert transaction["spans"][0]["tags"]["status"] == "internal_error" assert transaction["contexts"]["trace"]["status"] == "internal_error" @@ -985,9 +935,7 @@ def _llm_type(self): def _identifying_params(self): return {} - sentry_init( - integrations=[LangchainIntegration()], _experiments={"gen_ai_as_v2_spans": True} - ) + sentry_init(integrations=[LangchainIntegration()]) # Create a manual SentryLangchainCallback manual_callback = SentryLangchainCallback( @@ -1152,7 +1100,7 @@ def test_langchain_callback_list_existing_callback(sentry_init): assert handler is sentry_callback -def test_langchain_message_role_mapping(sentry_init, capture_items): +def test_langchain_message_role_mapping(sentry_init, capture_events): """Test that message roles are properly normalized in langchain integration.""" global llm_type llm_type = "openai-chat" @@ -1162,7 +1110,7 @@ def test_langchain_message_role_mapping(sentry_init, capture_items): traces_sample_rate=1.0, send_default_pii=True, ) - items = capture_items("transaction", "span") + events = capture_events() prompt = ChatPromptTemplate.from_messages( [ @@ -1198,18 +1146,19 @@ def test_langchain_message_role_mapping(sentry_init, capture_items): with start_transaction(): list(agent_executor.stream({"input": test_input})) - spans = [item.payload for item in items if item.type == "span"] + assert len(events) > 0 + tx = events[0] + assert tx["type"] == "transaction" + # Find spans with gen_ai operation that should have message data gen_ai_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op", "").startswith("gen_ai") + span for span in tx.get("spans", []) if span.get("op", "").startswith("gen_ai") ] # Check if any span has message data with normalized roles message_data_found = False for span in gen_ai_spans: - span_data = span.get("attributes", {}) + span_data = span.get("data", {}) if SPANDATA.GEN_AI_REQUEST_MESSAGES in span_data: message_data_found = True messages_data = span_data[SPANDATA.GEN_AI_REQUEST_MESSAGES] @@ -1290,6 +1239,84 @@ def test_langchain_message_role_normalization_units(): assert normalized[5] == "string message" # String message unchanged +def test_langchain_message_truncation(sentry_init, capture_events): + """Test that large messages are truncated properly in Langchain integration.""" + from langchain_core.outputs import LLMResult, Generation + + sentry_init( + integrations=[LangchainIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + callback = SentryLangchainCallback(max_span_map_size=100, include_prompts=True) + + run_id = "12345678-1234-1234-1234-123456789012" + serialized = {"_type": "openai-chat", "model_name": "gpt-3.5-turbo"} + + large_content = ( + "This is a very long message that will exceed our size limits. " * 1000 + ) + prompts = [ + "small message 1", + large_content, + large_content, + "small message 4", + "small message 5", + ] + + with start_transaction(): + callback.on_llm_start( + serialized=serialized, + prompts=prompts, + run_id=run_id, + name="my_pipeline", + invocation_params={ + "temperature": 0.7, + "max_tokens": 100, + "model": "gpt-3.5-turbo", + }, + ) + + response = LLMResult( + generations=[[Generation(text="The response")]], + llm_output={ + "token_usage": { + "total_tokens": 25, + "prompt_tokens": 10, + "completion_tokens": 15, + } + }, + ) + callback.on_llm_end(response=response, run_id=run_id) + + assert len(events) > 0 + tx = events[0] + assert tx["type"] == "transaction" + + llm_spans = [ + span + for span in tx.get("spans", []) + if span.get("op") == "gen_ai.text_completion" + ] + assert len(llm_spans) > 0 + + llm_span = llm_spans[0] + assert llm_span["data"]["gen_ai.operation.name"] == "text_completion" + assert llm_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == "my_pipeline" + + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in llm_span["data"] + messages_data = llm_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert isinstance(messages_data, str) + + parsed_messages = json.loads(messages_data) + assert isinstance(parsed_messages, list) + assert len(parsed_messages) == 1 + assert "small message 5" in str(parsed_messages[0]) + assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 + + @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -1300,7 +1327,7 @@ def test_langchain_message_role_normalization_units(): ], ) def test_langchain_embeddings_sync( - sentry_init, capture_items, send_default_pii, include_prompts + sentry_init, capture_events, send_default_pii, include_prompts ): """Test that sync embedding methods (embed_documents, embed_query) are properly traced.""" try: @@ -1313,7 +1340,7 @@ def test_langchain_embeddings_sync( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - items = capture_items("transaction", "span") + events = capture_events() # Mock the actual API call with mock.patch.object( @@ -1335,28 +1362,27 @@ def test_langchain_embeddings_sync( assert len(result) == 2 mock_embed_documents.assert_called_once() - spans = [item.payload for item in items if item.type == "span"] + # Check captured events + assert len(events) >= 1 + tx = events[0] + assert tx["type"] == "transaction" + # Find embeddings span embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 1 embeddings_span = embeddings_spans[0] - assert embeddings_span["name"] == "embeddings text-embedding-ada-002" - assert embeddings_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["attributes"]["gen_ai.request.model"] - == "text-embedding-ada-002" - ) + assert embeddings_span["description"] == "embeddings text-embedding-ada-002" + assert embeddings_span["origin"] == "auto.ai.langchain" + assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" + assert embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" # Check if input is captured based on PII settings if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] - input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] + input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] # Could be serialized as string if isinstance(input_data, str): assert "Hello world" in input_data @@ -1365,9 +1391,7 @@ def test_langchain_embeddings_sync( assert "Hello world" in input_data assert "Test document" in input_data else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( - "attributes", {} - ) + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get("data", {}) @pytest.mark.parametrize( @@ -1378,7 +1402,7 @@ def test_langchain_embeddings_sync( ], ) def test_langchain_embeddings_embed_query( - sentry_init, capture_items, send_default_pii, include_prompts + sentry_init, capture_events, send_default_pii, include_prompts ): """Test that embed_query method is properly traced.""" try: @@ -1391,7 +1415,7 @@ def test_langchain_embeddings_embed_query( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - items = capture_items("transaction", "span") + events = capture_events() # Mock the actual API call with mock.patch.object( @@ -1412,35 +1436,32 @@ def test_langchain_embeddings_embed_query( assert len(result) == 3 mock_embed_query.assert_called_once() - spans = [item.payload for item in items if item.type == "span"] + # Check captured events + assert len(events) >= 1 + tx = events[0] + assert tx["type"] == "transaction" + # Find embeddings span embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 1 embeddings_span = embeddings_spans[0] - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["attributes"]["gen_ai.request.model"] - == "text-embedding-ada-002" - ) + assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" + assert embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" # Check if input is captured based on PII settings if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] - input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] + input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] # Could be serialized as string if isinstance(input_data, str): assert "What is the capital of France?" in input_data else: assert "What is the capital of France?" in input_data else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( - "attributes", {} - ) + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get("data", {}) @pytest.mark.parametrize( @@ -1452,7 +1473,7 @@ def test_langchain_embeddings_embed_query( ) @pytest.mark.asyncio async def test_langchain_embeddings_async( - sentry_init, capture_items, send_default_pii, include_prompts + sentry_init, capture_events, send_default_pii, include_prompts ): """Test that async embedding methods (aembed_documents, aembed_query) are properly traced.""" try: @@ -1465,7 +1486,7 @@ async def test_langchain_embeddings_async( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - items = capture_items("transaction", "span") + events = capture_events() async def mock_aembed_documents(self, texts): return [[0.1, 0.2, 0.3] for _ in texts] @@ -1491,41 +1512,38 @@ async def mock_aembed_documents(self, texts): assert len(result) == 2 mock_aembed.assert_called_once() - spans = [item.payload for item in items if item.type == "span"] + # Check captured events + assert len(events) >= 1 + tx = events[0] + assert tx["type"] == "transaction" + # Find embeddings span embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 1 embeddings_span = embeddings_spans[0] - assert embeddings_span["name"] == "embeddings text-embedding-ada-002" - assert embeddings_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["attributes"]["gen_ai.request.model"] - == "text-embedding-ada-002" - ) + assert embeddings_span["description"] == "embeddings text-embedding-ada-002" + assert embeddings_span["origin"] == "auto.ai.langchain" + assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" + assert embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" # Check if input is captured based on PII settings if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] - input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] + input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] # Could be serialized as string if isinstance(input_data, str): assert "Async hello" in input_data or "Async test document" in input_data else: assert "Async hello" in input_data or "Async test document" in input_data else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( - "attributes", {} - ) + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get("data", {}) @pytest.mark.asyncio -async def test_langchain_embeddings_aembed_query(sentry_init, capture_items): +async def test_langchain_embeddings_aembed_query(sentry_init, capture_events): """Test that aembed_query method is properly traced.""" try: from langchain_openai import OpenAIEmbeddings @@ -1537,7 +1555,7 @@ async def test_langchain_embeddings_aembed_query(sentry_init, capture_items): traces_sample_rate=1.0, send_default_pii=True, ) - items = capture_items("transaction", "span") + events = capture_events() async def mock_aembed_query(self, text): return [0.1, 0.2, 0.3] @@ -1561,25 +1579,24 @@ async def mock_aembed_query(self, text): assert len(result) == 3 mock_aembed.assert_called_once() - spans = [item.payload for item in items if item.type == "span"] + # Check captured events + assert len(events) >= 1 + tx = events[0] + assert tx["type"] == "transaction" + # Find embeddings span embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 1 embeddings_span = embeddings_spans[0] - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert ( - embeddings_span["attributes"]["gen_ai.request.model"] - == "text-embedding-ada-002" - ) + assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" + assert embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" # Check if input is captured - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] - input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] + input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] # Could be serialized as string if isinstance(input_data, str): assert "Async query test" in input_data @@ -1587,7 +1604,7 @@ async def mock_aembed_query(self, text): assert "Async query test" in input_data -def test_langchain_embeddings_no_model_name(sentry_init, capture_items): +def test_langchain_embeddings_no_model_name(sentry_init, capture_events): """Test embeddings when model name is not available.""" try: from langchain_openai import OpenAIEmbeddings @@ -1598,7 +1615,7 @@ def test_langchain_embeddings_no_model_name(sentry_init, capture_items): integrations=[LangchainIntegration(include_prompts=False)], traces_sample_rate=1.0, ) - items = capture_items("transaction", "span") + events = capture_events() # Mock the actual API call and remove model attribute with mock.patch.object( @@ -1618,26 +1635,28 @@ def test_langchain_embeddings_no_model_name(sentry_init, capture_items): with start_transaction(name="test_embeddings_no_model"): embeddings.embed_documents(["Test"]) - spans = [item.payload for item in items if item.type == "span"] + # Check captured events + assert len(events) >= 1 + tx = events[0] + assert tx["type"] == "transaction" + # Find embeddings span embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 1 embeddings_span = embeddings_spans[0] - assert embeddings_span["name"] == "embeddings" - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert embeddings_span["description"] == "embeddings" + assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" # Model name should not be set if not available assert ( - "gen_ai.request.model" not in embeddings_span["attributes"] - or embeddings_span["attributes"]["gen_ai.request.model"] is None + "gen_ai.request.model" not in embeddings_span["data"] + or embeddings_span["data"]["gen_ai.request.model"] is None ) -def test_langchain_embeddings_integration_disabled(sentry_init, capture_items): +def test_langchain_embeddings_integration_disabled(sentry_init, capture_events): """Test that embeddings are not traced when integration is disabled.""" try: from langchain_openai import OpenAIEmbeddings @@ -1645,8 +1664,8 @@ def test_langchain_embeddings_integration_disabled(sentry_init, capture_items): pytest.skip("langchain_openai not installed") # Initialize without LangchainIntegration - sentry_init(traces_sample_rate=1.0, _experiments={"gen_ai_as_v2_spans": True}) - items = capture_items("transaction", "span") + sentry_init(traces_sample_rate=1.0) + events = capture_events() with mock.patch.object( OpenAIEmbeddings, @@ -1661,17 +1680,18 @@ def test_langchain_embeddings_integration_disabled(sentry_init, capture_items): embeddings.embed_documents(["Test"]) # Check that no embeddings spans were created - spans = [item.payload for item in items if item.type == "span"] - embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" - ] - # Should be empty since integration is disabled - assert len(embeddings_spans) == 0 + if events: + tx = events[0] + embeddings_spans = [ + span + for span in tx.get("spans", []) + if span.get("op") == "gen_ai.embeddings" + ] + # Should be empty since integration is disabled + assert len(embeddings_spans) == 0 -def test_langchain_embeddings_multiple_providers(sentry_init, capture_items): +def test_langchain_embeddings_multiple_providers(sentry_init, capture_events): """Test that embeddings work with different providers.""" try: from langchain_openai import OpenAIEmbeddings, AzureOpenAIEmbeddings @@ -1683,7 +1703,7 @@ def test_langchain_embeddings_multiple_providers(sentry_init, capture_items): traces_sample_rate=1.0, send_default_pii=True, ) - items = capture_items("transaction", "span") + events = capture_events() # Mock both providers with mock.patch.object( @@ -1711,24 +1731,26 @@ def test_langchain_embeddings_multiple_providers(sentry_init, capture_items): openai_embeddings.embed_documents(["OpenAI test"]) azure_embeddings.embed_documents(["Azure test"]) - spans = [item.payload for item in items if item.type == "span"] + # Check captured events + assert len(events) >= 1 + tx = events[0] + assert tx["type"] == "transaction" + # Find embeddings spans embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" ] # Should have 2 spans, one for each provider assert len(embeddings_spans) == 2 # Verify both spans have proper data for span in embeddings_spans: - assert span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert span["attributes"]["gen_ai.request.model"] == "text-embedding-ada-002" - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] + assert span["data"]["gen_ai.operation.name"] == "embeddings" + assert span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["data"] -def test_langchain_embeddings_error_handling(sentry_init, capture_items): +def test_langchain_embeddings_error_handling(sentry_init, capture_events): """Test that errors in embeddings are properly captured.""" try: from langchain_openai import OpenAIEmbeddings @@ -1740,7 +1762,7 @@ def test_langchain_embeddings_error_handling(sentry_init, capture_items): traces_sample_rate=1.0, send_default_pii=True, ) - items = capture_items("transaction", "span") + events = capture_events() # Mock the API call to raise an error with mock.patch.object( @@ -1759,16 +1781,15 @@ def test_langchain_embeddings_error_handling(sentry_init, capture_items): with pytest.raises(ValueError): embeddings.embed_documents(["Test"]) - [ - item.payload - for item in items - if item.type == "event" and item.payload.get("level") == "error" - ] + # The error should be captured + assert len(events) >= 1 + # We should have both the transaction and potentially an error event + [e for e in events if e.get("level") == "error"] # Note: errors might not be auto-captured depending on SDK settings, # but the span should still be created -def test_langchain_embeddings_multiple_calls(sentry_init, capture_items): +def test_langchain_embeddings_multiple_calls(sentry_init, capture_events): """Test that multiple embeddings calls within a transaction are all traced.""" try: from langchain_openai import OpenAIEmbeddings @@ -1780,7 +1801,7 @@ def test_langchain_embeddings_multiple_calls(sentry_init, capture_items): traces_sample_rate=1.0, send_default_pii=True, ) - items = capture_items("transaction", "span") + events = capture_events() # Mock the actual API calls with mock.patch.object( @@ -1807,31 +1828,32 @@ def test_langchain_embeddings_multiple_calls(sentry_init, capture_items): # Call embed_documents again embeddings.embed_documents(["Third batch"]) - spans = [item.payload for item in items if item.type == "span"] + # Check captured events + assert len(events) >= 1 + tx = events[0] + assert tx["type"] == "transaction" + # Find embeddings spans - should have 3 (2 embed_documents + 1 embed_query) embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 3 # Verify all spans have proper data for span in embeddings_spans: - assert span["attributes"]["gen_ai.operation.name"] == "embeddings" - assert span["attributes"]["gen_ai.request.model"] == "text-embedding-ada-002" - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] + assert span["data"]["gen_ai.operation.name"] == "embeddings" + assert span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["data"] # Verify the input data is different for each span input_data_list = [ - span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] - for span in embeddings_spans + span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] for span in embeddings_spans ] # They should all be different (different inputs) assert len(set(str(data) for data in input_data_list)) == 3 -def test_langchain_embeddings_span_hierarchy(sentry_init, capture_items): +def test_langchain_embeddings_span_hierarchy(sentry_init, capture_events): """Test that embeddings spans are properly nested within parent spans.""" try: from langchain_openai import OpenAIEmbeddings @@ -1843,7 +1865,7 @@ def test_langchain_embeddings_span_hierarchy(sentry_init, capture_items): traces_sample_rate=1.0, send_default_pii=True, ) - items = capture_items("transaction", "span") + events = capture_events() # Mock the actual API call with mock.patch.object( @@ -1862,15 +1884,15 @@ def test_langchain_embeddings_span_hierarchy(sentry_init, capture_items): with sentry_sdk.start_span(op="custom", name="custom operation"): embeddings.embed_documents(["Test within custom span"]) - spans = [item.payload for item in items if item.type == "span"] + # Check captured events + assert len(events) >= 1 + tx = events[0] + assert tx["type"] == "transaction" + # Find all spans embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" ] - - tx = next(item.payload for item in items if item.type == "transaction") custom_spans = [span for span in tx.get("spans", []) if span.get("op") == "custom"] assert len(embeddings_spans) == 1 @@ -1880,11 +1902,11 @@ def test_langchain_embeddings_span_hierarchy(sentry_init, capture_items): embeddings_span = embeddings_spans[0] custom_span = custom_spans[0] - assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" assert custom_span["description"] == "custom operation" -def test_langchain_embeddings_with_list_and_string_inputs(sentry_init, capture_items): +def test_langchain_embeddings_with_list_and_string_inputs(sentry_init, capture_events): """Test that embeddings correctly handle both list and string inputs.""" try: from langchain_openai import OpenAIEmbeddings @@ -1896,7 +1918,7 @@ def test_langchain_embeddings_with_list_and_string_inputs(sentry_init, capture_i traces_sample_rate=1.0, send_default_pii=True, ) - items = capture_items("transaction", "span") + events = capture_events() # Mock the actual API calls with mock.patch.object( @@ -1921,19 +1943,21 @@ def test_langchain_embeddings_with_list_and_string_inputs(sentry_init, capture_i # embed_query takes a string embeddings.embed_query("Single string query") - spans = [item.payload for item in items if item.type == "span"] + # Check captured events + assert len(events) >= 1 + tx = events[0] + assert tx["type"] == "transaction" + # Find embeddings spans embeddings_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 2 # Both should have input data captured as lists for span in embeddings_spans: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] - input_data = span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["data"] + input_data = span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] # Input should be normalized to list format if isinstance(input_data, str): # If serialized, should contain the input text @@ -1951,7 +1975,7 @@ def test_langchain_embeddings_with_list_and_string_inputs(sentry_init, capture_i ) def test_langchain_response_model_extraction( sentry_init, - capture_items, + capture_events, response_metadata_model, expected_model, ): @@ -1960,7 +1984,7 @@ def test_langchain_response_model_extraction( traces_sample_rate=1.0, send_default_pii=True, ) - items = capture_items("transaction", "span") + events = capture_events() callback = SentryLangchainCallback(max_span_map_size=100, include_prompts=True) @@ -1985,22 +2009,236 @@ def test_langchain_response_model_extraction( response = Mock(generations=[[generation]]) callback.on_llm_end(response=response, run_id=run_id) - spans = [item.payload for item in items if item.type == "span"] + assert len(events) > 0 + tx = events[0] + assert tx["type"] == "transaction" + llm_spans = [ span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.text_completion" + for span in tx.get("spans", []) + if span.get("op") == "gen_ai.text_completion" ] assert len(llm_spans) > 0 llm_span = llm_spans[0] - assert llm_span["attributes"]["gen_ai.operation.name"] == "text_completion" + assert llm_span["data"]["gen_ai.operation.name"] == "text_completion" if expected_model is not None: - assert SPANDATA.GEN_AI_RESPONSE_MODEL in llm_span["attributes"] - assert llm_span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == expected_model + assert SPANDATA.GEN_AI_RESPONSE_MODEL in llm_span["data"] + assert llm_span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == expected_model else: - assert SPANDATA.GEN_AI_RESPONSE_MODEL not in llm_span.get("attributes", {}) + assert SPANDATA.GEN_AI_RESPONSE_MODEL not in llm_span.get("data", {}) + + +# Tests for multimodal content transformation functions + + +class TestTransformLangchainContentBlock: + """Tests for _transform_langchain_content_block function.""" + + def test_transform_image_base64(self): + """Test transformation of base64-encoded image content.""" + content_block = { + "type": "image", + "base64": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + "mime_type": "image/jpeg", + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + + def test_transform_image_url(self): + """Test transformation of URL-referenced image content.""" + content_block = { + "type": "image", + "url": "https://example.com/image.jpg", + "mime_type": "image/jpeg", + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "image/jpeg", + "uri": "https://example.com/image.jpg", + } + + def test_transform_image_file_id(self): + """Test transformation of file_id-referenced image content.""" + content_block = { + "type": "image", + "file_id": "file-abc123", + "mime_type": "image/png", + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "file", + "modality": "image", + "mime_type": "image/png", + "file_id": "file-abc123", + } + + def test_transform_image_url_legacy_with_data_uri(self): + """Test transformation of legacy image_url format with data: URI (base64).""" + content_block = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD"}, + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRgABAQAAAQABAAD", + } + + def test_transform_image_url_legacy_with_http_url(self): + """Test transformation of legacy image_url format with HTTP URL.""" + content_block = { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": "https://example.com/image.png", + } + + def test_transform_image_url_legacy_string_url(self): + """Test transformation of legacy image_url format with string URL.""" + content_block = { + "type": "image_url", + "image_url": "https://example.com/image.gif", + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "", + "uri": "https://example.com/image.gif", + } + + def test_transform_image_url_legacy_data_uri_png(self): + """Test transformation of legacy image_url format with PNG data URI.""" + content_block = { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + }, + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/png", + "content": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + } + + def test_transform_missing_mime_type(self): + """Test transformation when mime_type is not provided.""" + content_block = { + "type": "image", + "base64": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "", + "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + + def test_transform_anthropic_source_base64(self): + """Test transformation of Anthropic-style image with base64 source.""" + content_block = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAE...", + }, + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/png", + "content": "iVBORw0KGgoAAAANSUhEUgAAAAE...", + } + + def test_transform_anthropic_source_url(self): + """Test transformation of Anthropic-style image with URL source.""" + content_block = { + "type": "image", + "source": { + "type": "url", + "media_type": "image/jpeg", + "url": "https://example.com/image.jpg", + }, + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "image/jpeg", + "uri": "https://example.com/image.jpg", + } + + def test_transform_anthropic_source_without_media_type(self): + """Test transformation of Anthropic-style image without media_type uses empty mime_type.""" + content_block = { + "type": "image", + "mime_type": "image/webp", # Top-level mime_type is ignored by standard Anthropic format + "source": { + "type": "base64", + "data": "UklGRh4AAABXRUJQVlA4IBIAAAAwAQCdASoBAAEAAQAcJYgCdAEO", + }, + } + result = _transform_langchain_content_block(content_block) + # Note: The shared transform_content_part uses media_type from source, not top-level mime_type + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "", + "content": "UklGRh4AAABXRUJQVlA4IBIAAAAwAQCdASoBAAEAAQAcJYgCdAEO", + } + + def test_transform_google_inline_data(self): + """Test transformation of Google-style inline_data format.""" + content_block = { + "inline_data": { + "mime_type": "image/jpeg", + "data": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + + def test_transform_google_file_data(self): + """Test transformation of Google-style file_data format.""" + content_block = { + "file_data": { + "mime_type": "image/png", + "file_uri": "gs://bucket/path/to/image.png", + } + } + result = _transform_langchain_content_block(content_block) + assert result == { + "type": "uri", + "modality": "image", + "mime_type": "image/png", + "uri": "gs://bucket/path/to/image.png", + } @pytest.mark.parametrize( @@ -2048,13 +2286,13 @@ def test_langchain_response_model_extraction( ], ) def test_langchain_ai_system_detection( - sentry_init, capture_items, ai_type, expected_system + sentry_init, capture_events, ai_type, expected_system ): sentry_init( integrations=[LangchainIntegration()], traces_sample_rate=1.0, ) - items = capture_items("transaction", "span") + events = capture_events() callback = SentryLangchainCallback(max_span_map_size=100, include_prompts=True) @@ -2074,17 +2312,136 @@ def test_langchain_ai_system_detection( response = Mock(generations=[[generation]]) callback.on_llm_end(response=response, run_id=run_id) - spans = [item.payload for item in items if item.type == "span"] + assert len(events) > 0 + tx = events[0] + assert tx["type"] == "transaction" + llm_spans = [ span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.text_completion" + for span in tx.get("spans", []) + if span.get("op") == "gen_ai.text_completion" ] assert len(llm_spans) > 0 llm_span = llm_spans[0] if expected_system is not None: - assert llm_span["attributes"][SPANDATA.GEN_AI_SYSTEM] == expected_system + assert llm_span["data"][SPANDATA.GEN_AI_SYSTEM] == expected_system else: - assert SPANDATA.GEN_AI_SYSTEM not in llm_span.get("attributes", {}) + assert SPANDATA.GEN_AI_SYSTEM not in llm_span.get("data", {}) + + +class TestTransformLangchainMessageContent: + """Tests for _transform_langchain_message_content function.""" + + def test_transform_string_content(self): + """Test that string content is returned unchanged.""" + result = _transform_langchain_message_content("Hello, world!") + assert result == "Hello, world!" + + def test_transform_list_with_text_blocks(self): + """Test transformation of list with text blocks (unchanged).""" + content = [ + {"type": "text", "text": "First message"}, + {"type": "text", "text": "Second message"}, + ] + result = _transform_langchain_message_content(content) + assert result == content + + def test_transform_list_with_image_blocks(self): + """Test transformation of list containing image blocks.""" + content = [ + {"type": "text", "text": "Check out this image:"}, + { + "type": "image", + "base64": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + "mime_type": "image/jpeg", + }, + ] + result = _transform_langchain_message_content(content) + assert len(result) == 2 + assert result[0] == {"type": "text", "text": "Check out this image:"} + assert result[1] == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", + } + + def test_transform_list_with_mixed_content(self): + """Test transformation of list with mixed content types.""" + content = [ + {"type": "text", "text": "Here are some files:"}, + { + "type": "image", + "url": "https://example.com/image.jpg", + "mime_type": "image/jpeg", + }, + { + "type": "file", + "file_id": "doc-123", + "mime_type": "application/pdf", + }, + {"type": "audio", "base64": "audio_data...", "mime_type": "audio/mp3"}, + ] + result = _transform_langchain_message_content(content) + assert len(result) == 4 + assert result[0] == {"type": "text", "text": "Here are some files:"} + assert result[1] == { + "type": "uri", + "modality": "image", + "mime_type": "image/jpeg", + "uri": "https://example.com/image.jpg", + } + assert result[2] == { + "type": "file", + "modality": "document", + "mime_type": "application/pdf", + "file_id": "doc-123", + } + assert result[3] == { + "type": "blob", + "modality": "audio", + "mime_type": "audio/mp3", + "content": "audio_data...", + } + + def test_transform_list_with_non_dict_items(self): + """Test transformation handles non-dict items in list.""" + content = ["plain string", {"type": "text", "text": "dict text"}] + result = _transform_langchain_message_content(content) + assert result == ["plain string", {"type": "text", "text": "dict text"}] + + def test_transform_tuple_content(self): + """Test transformation of tuple content.""" + content = ( + {"type": "text", "text": "Message"}, + {"type": "image", "base64": "data...", "mime_type": "image/png"}, + ) + result = _transform_langchain_message_content(content) + assert len(result) == 2 + assert result[1] == { + "type": "blob", + "modality": "image", + "mime_type": "image/png", + "content": "data...", + } + + def test_transform_list_with_legacy_image_url(self): + """Test transformation of list containing legacy image_url blocks.""" + content = [ + {"type": "text", "text": "Check this:"}, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ..."}, + }, + ] + result = _transform_langchain_message_content(content) + assert len(result) == 2 + assert result[0] == {"type": "text", "text": "Check this:"} + assert result[1] == { + "type": "blob", + "modality": "image", + "mime_type": "image/jpeg", + "content": "/9j/4AAQ...", + } From af25aff665715a5e63d39c8604e3de9da736f977 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 21 Apr 2026 08:44:08 +0200 Subject: [PATCH 4/5] revert langchain changes --- sentry_sdk/integrations/langchain.py | 38 +- .../integrations/langchain/test_langchain.py | 925 ++++++------------ 2 files changed, 324 insertions(+), 639 deletions(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 49fa04c034..52a7fe6695 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -15,7 +15,6 @@ normalize_message_roles, set_data_normalized, truncate_and_annotate_messages, - transform_content_part, ) from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration @@ -129,39 +128,6 @@ def _get_ai_system(all_params: "Dict[str, Any]") -> "Optional[str]": } -def _transform_langchain_content_block( - content_block: "Dict[str, Any]", -) -> "Dict[str, Any]": - """ - Transform a LangChain content block using the shared transform_content_part function. - - Returns the original content block if transformation is not applicable - (e.g., for text blocks or unrecognized formats). - """ - result = transform_content_part(content_block) - return result if result is not None else content_block - - -def _transform_langchain_message_content(content: "Any") -> "Any": - """ - Transform LangChain message content, handling both string content and - list of content blocks. - """ - if isinstance(content, str): - return content - - if isinstance(content, (list, tuple)): - transformed = [] - for block in content: - if isinstance(block, dict): - transformed.append(_transform_langchain_content_block(block)) - else: - transformed.append(block) - return transformed - - return content - - # Contextvar to track agent names in a stack for re-entrant agent support _agent_stack: "contextvars.ContextVar[Optional[List[Optional[str]]]]" = ( contextvars.ContextVar("langchain_agent_stack", default=None) @@ -313,9 +279,7 @@ def _handle_error(self, run_id: "UUID", error: "Any") -> None: del self.span_map[run_id] def _normalize_langchain_message(self, message: "BaseMessage") -> "Any": - # Transform content to handle multimodal data (images, audio, video, files) - transformed_content = _transform_langchain_message_content(message.content) - parsed = {"role": message.type, "content": transformed_content} + parsed = {"role": message.type, "content": message.content} parsed.update(message.additional_kwargs) return parsed diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 498a5d6f4a..319b96a06a 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -27,8 +27,6 @@ from sentry_sdk.integrations.langchain import ( LangchainIntegration, SentryLangchainCallback, - _transform_langchain_content_block, - _transform_langchain_message_content, ) try: @@ -97,7 +95,7 @@ def _llm_type(self) -> str: def test_langchain_text_completion( sentry_init, - capture_events, + capture_items, get_model_response, ): sentry_init( @@ -109,7 +107,7 @@ def test_langchain_text_completion( traces_sample_rate=1.0, send_default_pii=True, ) - events = capture_events() + items = capture_items("transaction", "span") model_response = get_model_response( Completion( @@ -149,25 +147,29 @@ def test_langchain_text_completion( input_text = "What is the capital of France?" model.invoke(input_text, config={"run_name": "my-snazzy-pipeline"}) - tx = events[0] + tx = next(item.payload for item in items if item.type == "transaction") assert tx["type"] == "transaction" + spans = [item.payload for item in items if item.type == "span"] llm_spans = [ span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.text_completion" + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.text_completion" ] assert len(llm_spans) > 0 llm_span = llm_spans[0] - assert llm_span["description"] == "text_completion gpt-3.5-turbo" - assert llm_span["data"]["gen_ai.system"] == "openai" - assert llm_span["data"]["gen_ai.pipeline.name"] == "my-snazzy-pipeline" - assert llm_span["data"]["gen_ai.request.model"] == "gpt-3.5-turbo" - assert llm_span["data"]["gen_ai.response.text"] == "The capital of France is Paris." - assert llm_span["data"]["gen_ai.usage.total_tokens"] == 25 - assert llm_span["data"]["gen_ai.usage.input_tokens"] == 10 - assert llm_span["data"]["gen_ai.usage.output_tokens"] == 15 + assert llm_span["name"] == "text_completion gpt-3.5-turbo" + assert llm_span["attributes"]["gen_ai.system"] == "openai" + assert llm_span["attributes"]["gen_ai.pipeline.name"] == "my-snazzy-pipeline" + assert llm_span["attributes"]["gen_ai.request.model"] == "gpt-3.5-turbo" + assert ( + llm_span["attributes"]["gen_ai.response.text"] + == "The capital of France is Paris." + ) + assert llm_span["attributes"]["gen_ai.usage.total_tokens"] == 25 + assert llm_span["attributes"]["gen_ai.usage.input_tokens"] == 10 + assert llm_span["attributes"]["gen_ai.usage.output_tokens"] == 15 @pytest.mark.skipif( @@ -196,7 +198,7 @@ def test_langchain_text_completion( ) def test_langchain_create_agent( sentry_init, - capture_events, + capture_items, send_default_pii, include_prompts, system_instructions_content, @@ -213,7 +215,7 @@ def test_langchain_create_agent( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - events = capture_events() + items = capture_items("transaction", "span") model_response = get_model_response( nonstreaming_responses_model_response, @@ -250,22 +252,23 @@ def test_langchain_create_agent( }, ) - tx = events[0] + tx = next(item.payload for item in items if item.type == "transaction") assert tx["type"] == "transaction" assert tx["contexts"]["trace"]["origin"] == "manual" - chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") + spans = [item.payload for item in items if item.type == "span"] + chat_spans = list(x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat") assert len(chat_spans) == 1 - assert chat_spans[0]["origin"] == "auto.ai.langchain" + assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[0]["data"]["gen_ai.system"] == "openai-chat" - assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 10 - assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 20 - assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 30 + assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 10 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 20 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30 if send_default_pii and include_prompts: assert ( - chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hello, how can I help you?" ) @@ -276,7 +279,9 @@ def test_langchain_create_agent( "type": "text", "content": "You are very powerful assistant, but don't know current events", } - ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + ] == json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) else: assert [ { @@ -287,11 +292,17 @@ def test_langchain_create_agent( "type": "text", "content": "Be concise and clear.", }, - ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + ] == json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {}) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) @pytest.mark.skipif( @@ -309,7 +320,7 @@ def test_langchain_create_agent( ) def test_tool_execution_span( sentry_init, - capture_events, + capture_items, send_default_pii, include_prompts, get_model_response, @@ -324,7 +335,7 @@ def test_tool_execution_span( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - events = capture_events() + items = capture_items("transaction", "span") responses = responses_tool_call_model_responses( tool_name="get_word_length", @@ -400,60 +411,71 @@ def test_tool_execution_span( }, ) - tx = events[0] + tx = next(item.payload for item in items if item.type == "transaction") assert tx["type"] == "transaction" assert tx["contexts"]["trace"]["origin"] == "manual" - chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") + spans = [item.payload for item in items if item.type == "span"] + chat_spans = list(x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat") assert len(chat_spans) == 2 - tool_exec_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool") + tool_exec_spans = list( + x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.execute_tool" + ) assert len(tool_exec_spans) == 1 tool_exec_span = tool_exec_spans[0] - assert chat_spans[0]["origin"] == "auto.ai.langchain" - assert chat_spans[1]["origin"] == "auto.ai.langchain" - assert tool_exec_span["origin"] == "auto.ai.langchain" + assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192 - assert chat_spans[0]["data"]["gen_ai.system"] == "openai-chat" + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 + assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" - assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117 - assert chat_spans[1]["data"]["gen_ai.system"] == "openai-chat" + assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 + assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 + assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 + assert chat_spans[1]["attributes"]["gen_ai.system"] == "openai-chat" if send_default_pii and include_prompts: - assert "word" in tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_INPUT] + assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] - assert "5" in chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("data", {}), ( + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get( + "attributes", {} + ), ( "Tool calls should be recorded when send_default_pii=True and include_prompts=True" ) - tool_calls_data = chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + tool_calls_data = chat_spans[0]["attributes"][ + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS + ] assert isinstance(tool_calls_data, str) assert "get_word_length" in tool_calls_data else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("data", {}) - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("data", {}) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) # Verify tool calls are NOT recorded when PII is disabled assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( - "data", {} + "attributes", {} ), ( f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " f"and include_prompts={include_prompts}" ) assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( - "data", {} + "attributes", {} ), ( f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " f"and include_prompts={include_prompts}" @@ -461,7 +483,7 @@ def test_tool_execution_span( # Verify that available tools are always recorded regardless of PII settings for chat_span in chat_spans: - tools_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] + tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] assert "get_word_length" in tools_data @@ -488,7 +510,7 @@ def test_tool_execution_span( ) def test_langchain_openai_tools_agent( sentry_init, - capture_events, + capture_items, send_default_pii, include_prompts, system_instructions_content, @@ -505,7 +527,7 @@ def test_langchain_openai_tools_agent( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - events = capture_events() + items = capture_items("transaction", "span") prompt = ChatPromptTemplate.from_messages( [ @@ -700,40 +722,47 @@ def test_langchain_openai_tools_agent( with start_transaction(): list(agent_executor.stream({"input": "How many letters in the word eudca"})) - tx = events[0] + tx = next(item.payload for item in items if item.type == "transaction") assert tx["type"] == "transaction" assert tx["contexts"]["trace"]["origin"] == "manual" - invoke_agent_span = next(x for x in tx["spans"] if x["op"] == "gen_ai.invoke_agent") - chat_spans = list(x for x in tx["spans"] if x["op"] == "gen_ai.chat") - tool_exec_span = next(x for x in tx["spans"] if x["op"] == "gen_ai.execute_tool") + spans = [item.payload for item in items if item.type == "span"] + invoke_agent_span = next( + x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.invoke_agent" + ) + chat_spans = list(x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat") + tool_exec_span = next( + x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.execute_tool" + ) assert len(chat_spans) == 2 - assert invoke_agent_span["origin"] == "auto.ai.langchain" - assert chat_spans[0]["origin"] == "auto.ai.langchain" - assert chat_spans[1]["origin"] == "auto.ai.langchain" - assert tool_exec_span["origin"] == "auto.ai.langchain" + assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" # We can't guarantee anything about the "shape" of the langchain execution graph - assert len(list(x for x in tx["spans"] if x["op"] == "gen_ai.chat")) > 0 + assert ( + len(list(x for x in spans if x["attributes"]["sentry.op"] == "gen_ai.chat")) > 0 + ) # Token usage is only available in newer versions of langchain (v0.2+) # where usage_metadata is supported on AIMessageChunk - if "gen_ai.usage.input_tokens" in chat_spans[0]["data"]: - assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 192 + if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - if "gen_ai.usage.input_tokens" in chat_spans[1]["data"]: - assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["data"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["data"]["gen_ai.usage.total_tokens"] == 117 + if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: + assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 + assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 + assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 if send_default_pii and include_prompts: - assert "5" in chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - assert "word" in tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_INPUT] - assert 5 == int(tool_exec_span["data"][SPANDATA.GEN_AI_TOOL_OUTPUT]) + assert "5" in chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] + assert 5 == int(tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_OUTPUT]) param_id = request.node.callspec.id if "string" in param_id: @@ -742,7 +771,9 @@ def test_langchain_openai_tools_agent( "type": "text", "content": "You are very powerful assistant, but don't know current events", } - ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + ] == json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) else: assert [ { @@ -753,15 +784,21 @@ def test_langchain_openai_tools_agent( "type": "text", "content": "Be concise and clear.", }, - ] == json.loads(chat_spans[0]["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) + ] == json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) - assert "5" in chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("data", {}), ( + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get( + "attributes", {} + ), ( "Tool calls should be recorded when send_default_pii=True and include_prompts=True" ) - tool_calls_data = chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + tool_calls_data = chat_spans[0]["attributes"][ + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS + ] assert isinstance(tool_calls_data, (list, str)) # Could be serialized if isinstance(tool_calls_data, str): assert "get_word_length" in tool_calls_data @@ -770,45 +807,55 @@ def test_langchain_openai_tools_agent( tool_call_str = str(tool_calls_data) assert "get_word_length" in tool_call_str else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("data", {}) - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("data", {}) - assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("data", {}) - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("data", {}) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) # Verify tool calls are NOT recorded when PII is disabled assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( - "data", {} + "attributes", {} ), ( f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " f"and include_prompts={include_prompts}" ) assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( - "data", {} + "attributes", {} ), ( f"Tool calls should NOT be recorded when send_default_pii={send_default_pii} " f"and include_prompts={include_prompts}" ) # Verify finish_reasons is always an array of strings - assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ "function_call" ] - assert chat_spans[1]["data"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == ["stop"] + assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + "stop" + ] # Verify that available tools are always recorded regardless of PII settings for chat_span in chat_spans: - tools_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] + tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] assert tools_data is not None, ( "Available tools should always be recorded regardless of PII settings" ) assert "get_word_length" in tools_data -def test_langchain_error(sentry_init, capture_events): +def test_langchain_error(sentry_init, capture_items): global llm_type llm_type = "acme-llm" @@ -817,7 +864,7 @@ def test_langchain_error(sentry_init, capture_events): traces_sample_rate=1.0, send_default_pii=True, ) - events = capture_events() + items = capture_items("event", "transaction", "span") prompt = ChatPromptTemplate.from_messages( [ @@ -843,11 +890,11 @@ def test_langchain_error(sentry_init, capture_events): with start_transaction(), pytest.raises(ValueError): list(agent_executor.stream({"input": "How many letters in the word eudca"})) - error = events[0] + (error,) = (item.payload for item in items if item.type == "event") assert error["level"] == "error" -def test_span_status_error(sentry_init, capture_events): +def test_span_status_error(sentry_init, capture_items): global llm_type llm_type = "acme-llm" @@ -855,7 +902,7 @@ def test_span_status_error(sentry_init, capture_events): integrations=[LangchainIntegration(include_prompts=True)], traces_sample_rate=1.0, ) - events = capture_events() + items = capture_items("event", "transaction", "span") with start_transaction(name="test"): prompt = ChatPromptTemplate.from_messages( @@ -884,10 +931,13 @@ def test_span_status_error(sentry_init, capture_events): with pytest.raises(ValueError): list(agent_executor.stream({"input": "How many letters in the word eudca"})) - (error, transaction) = events + (error,) = (item.payload for item in items if item.type == "event") assert error["level"] == "error" - assert transaction["spans"][0]["status"] == "internal_error" - assert transaction["spans"][0]["tags"]["status"] == "internal_error" + + spans = [item.payload for item in items if item.type == "span"] + assert spans[0]["status"] == "error" + + (transaction,) = (item.payload for item in items if item.type == "transaction") assert transaction["contexts"]["trace"]["status"] == "internal_error" @@ -935,7 +985,9 @@ def _llm_type(self): def _identifying_params(self): return {} - sentry_init(integrations=[LangchainIntegration()]) + sentry_init( + integrations=[LangchainIntegration()], _experiments={"gen_ai_as_v2_spans": True} + ) # Create a manual SentryLangchainCallback manual_callback = SentryLangchainCallback( @@ -1100,7 +1152,7 @@ def test_langchain_callback_list_existing_callback(sentry_init): assert handler is sentry_callback -def test_langchain_message_role_mapping(sentry_init, capture_events): +def test_langchain_message_role_mapping(sentry_init, capture_items): """Test that message roles are properly normalized in langchain integration.""" global llm_type llm_type = "openai-chat" @@ -1110,7 +1162,7 @@ def test_langchain_message_role_mapping(sentry_init, capture_events): traces_sample_rate=1.0, send_default_pii=True, ) - events = capture_events() + items = capture_items("transaction", "span") prompt = ChatPromptTemplate.from_messages( [ @@ -1146,19 +1198,18 @@ def test_langchain_message_role_mapping(sentry_init, capture_events): with start_transaction(): list(agent_executor.stream({"input": test_input})) - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] # Find spans with gen_ai operation that should have message data gen_ai_spans = [ - span for span in tx.get("spans", []) if span.get("op", "").startswith("gen_ai") + span + for span in spans + if span["attributes"].get("sentry.op", "").startswith("gen_ai") ] # Check if any span has message data with normalized roles message_data_found = False for span in gen_ai_spans: - span_data = span.get("data", {}) + span_data = span.get("attributes", {}) if SPANDATA.GEN_AI_REQUEST_MESSAGES in span_data: message_data_found = True messages_data = span_data[SPANDATA.GEN_AI_REQUEST_MESSAGES] @@ -1239,7 +1290,7 @@ def test_langchain_message_role_normalization_units(): assert normalized[5] == "string message" # String message unchanged -def test_langchain_message_truncation(sentry_init, capture_events): +def test_langchain_message_truncation(sentry_init, capture_items): """Test that large messages are truncated properly in Langchain integration.""" from langchain_core.outputs import LLMResult, Generation @@ -1248,7 +1299,7 @@ def test_langchain_message_truncation(sentry_init, capture_events): traces_sample_rate=1.0, send_default_pii=True, ) - events = capture_events() + items = capture_items("transaction", "span") callback = SentryLangchainCallback(max_span_map_size=100, include_prompts=True) @@ -1291,23 +1342,23 @@ def test_langchain_message_truncation(sentry_init, capture_events): ) callback.on_llm_end(response=response, run_id=run_id) - assert len(events) > 0 - tx = events[0] + tx = next(item.payload for item in items if item.type == "transaction") assert tx["type"] == "transaction" + spans = [item.payload for item in items if item.type == "span"] llm_spans = [ span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.text_completion" + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.text_completion" ] assert len(llm_spans) > 0 llm_span = llm_spans[0] - assert llm_span["data"]["gen_ai.operation.name"] == "text_completion" - assert llm_span["data"][SPANDATA.GEN_AI_PIPELINE_NAME] == "my_pipeline" + assert llm_span["attributes"]["gen_ai.operation.name"] == "text_completion" + assert llm_span["attributes"][SPANDATA.GEN_AI_PIPELINE_NAME] == "my_pipeline" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in llm_span["data"] - messages_data = llm_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in llm_span["attributes"] + messages_data = llm_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert isinstance(messages_data, str) parsed_messages = json.loads(messages_data) @@ -1327,7 +1378,7 @@ def test_langchain_message_truncation(sentry_init, capture_events): ], ) def test_langchain_embeddings_sync( - sentry_init, capture_events, send_default_pii, include_prompts + sentry_init, capture_items, send_default_pii, include_prompts ): """Test that sync embedding methods (embed_documents, embed_query) are properly traced.""" try: @@ -1340,7 +1391,7 @@ def test_langchain_embeddings_sync( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - events = capture_events() + items = capture_items("transaction", "span") # Mock the actual API call with mock.patch.object( @@ -1362,27 +1413,28 @@ def test_langchain_embeddings_sync( assert len(result) == 2 mock_embed_documents.assert_called_once() - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] # Find embeddings span embeddings_spans = [ - span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 1 embeddings_span = embeddings_spans[0] - assert embeddings_span["description"] == "embeddings text-embedding-ada-002" - assert embeddings_span["origin"] == "auto.ai.langchain" - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" - assert embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" + assert embeddings_span["name"] == "embeddings text-embedding-ada-002" + assert embeddings_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert ( + embeddings_span["attributes"]["gen_ai.request.model"] + == "text-embedding-ada-002" + ) # Check if input is captured based on PII settings if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] - input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] + input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] # Could be serialized as string if isinstance(input_data, str): assert "Hello world" in input_data @@ -1391,7 +1443,9 @@ def test_langchain_embeddings_sync( assert "Hello world" in input_data assert "Test document" in input_data else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get("data", {}) + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( + "attributes", {} + ) @pytest.mark.parametrize( @@ -1402,7 +1456,7 @@ def test_langchain_embeddings_sync( ], ) def test_langchain_embeddings_embed_query( - sentry_init, capture_events, send_default_pii, include_prompts + sentry_init, capture_items, send_default_pii, include_prompts ): """Test that embed_query method is properly traced.""" try: @@ -1415,7 +1469,7 @@ def test_langchain_embeddings_embed_query( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - events = capture_events() + items = capture_items("transaction", "span") # Mock the actual API call with mock.patch.object( @@ -1436,32 +1490,35 @@ def test_langchain_embeddings_embed_query( assert len(result) == 3 mock_embed_query.assert_called_once() - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] # Find embeddings span embeddings_spans = [ - span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 1 embeddings_span = embeddings_spans[0] - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" - assert embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert ( + embeddings_span["attributes"]["gen_ai.request.model"] + == "text-embedding-ada-002" + ) # Check if input is captured based on PII settings if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] - input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] + input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] # Could be serialized as string if isinstance(input_data, str): assert "What is the capital of France?" in input_data else: assert "What is the capital of France?" in input_data else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get("data", {}) + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( + "attributes", {} + ) @pytest.mark.parametrize( @@ -1473,7 +1530,7 @@ def test_langchain_embeddings_embed_query( ) @pytest.mark.asyncio async def test_langchain_embeddings_async( - sentry_init, capture_events, send_default_pii, include_prompts + sentry_init, capture_items, send_default_pii, include_prompts ): """Test that async embedding methods (aembed_documents, aembed_query) are properly traced.""" try: @@ -1486,7 +1543,7 @@ async def test_langchain_embeddings_async( traces_sample_rate=1.0, send_default_pii=send_default_pii, ) - events = capture_events() + items = capture_items("transaction", "span") async def mock_aembed_documents(self, texts): return [[0.1, 0.2, 0.3] for _ in texts] @@ -1512,38 +1569,41 @@ async def mock_aembed_documents(self, texts): assert len(result) == 2 mock_aembed.assert_called_once() - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] # Find embeddings span embeddings_spans = [ - span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 1 embeddings_span = embeddings_spans[0] - assert embeddings_span["description"] == "embeddings text-embedding-ada-002" - assert embeddings_span["origin"] == "auto.ai.langchain" - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" - assert embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" + assert embeddings_span["name"] == "embeddings text-embedding-ada-002" + assert embeddings_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert ( + embeddings_span["attributes"]["gen_ai.request.model"] + == "text-embedding-ada-002" + ) # Check if input is captured based on PII settings if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] - input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] + input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] # Could be serialized as string if isinstance(input_data, str): assert "Async hello" in input_data or "Async test document" in input_data else: assert "Async hello" in input_data or "Async test document" in input_data else: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get("data", {}) + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT not in embeddings_span.get( + "attributes", {} + ) @pytest.mark.asyncio -async def test_langchain_embeddings_aembed_query(sentry_init, capture_events): +async def test_langchain_embeddings_aembed_query(sentry_init, capture_items): """Test that aembed_query method is properly traced.""" try: from langchain_openai import OpenAIEmbeddings @@ -1555,7 +1615,7 @@ async def test_langchain_embeddings_aembed_query(sentry_init, capture_events): traces_sample_rate=1.0, send_default_pii=True, ) - events = capture_events() + items = capture_items("transaction", "span") async def mock_aembed_query(self, text): return [0.1, 0.2, 0.3] @@ -1579,24 +1639,25 @@ async def mock_aembed_query(self, text): assert len(result) == 3 mock_aembed.assert_called_once() - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] # Find embeddings span embeddings_spans = [ - span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 1 embeddings_span = embeddings_spans[0] - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" - assert embeddings_span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert ( + embeddings_span["attributes"]["gen_ai.request.model"] + == "text-embedding-ada-002" + ) # Check if input is captured - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["data"] - input_data = embeddings_span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in embeddings_span["attributes"] + input_data = embeddings_span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] # Could be serialized as string if isinstance(input_data, str): assert "Async query test" in input_data @@ -1604,7 +1665,7 @@ async def mock_aembed_query(self, text): assert "Async query test" in input_data -def test_langchain_embeddings_no_model_name(sentry_init, capture_events): +def test_langchain_embeddings_no_model_name(sentry_init, capture_items): """Test embeddings when model name is not available.""" try: from langchain_openai import OpenAIEmbeddings @@ -1615,7 +1676,7 @@ def test_langchain_embeddings_no_model_name(sentry_init, capture_events): integrations=[LangchainIntegration(include_prompts=False)], traces_sample_rate=1.0, ) - events = capture_events() + items = capture_items("transaction", "span") # Mock the actual API call and remove model attribute with mock.patch.object( @@ -1635,28 +1696,26 @@ def test_langchain_embeddings_no_model_name(sentry_init, capture_events): with start_transaction(name="test_embeddings_no_model"): embeddings.embed_documents(["Test"]) - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] # Find embeddings span embeddings_spans = [ - span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 1 embeddings_span = embeddings_spans[0] - assert embeddings_span["description"] == "embeddings" - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" + assert embeddings_span["name"] == "embeddings" + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" # Model name should not be set if not available assert ( - "gen_ai.request.model" not in embeddings_span["data"] - or embeddings_span["data"]["gen_ai.request.model"] is None + "gen_ai.request.model" not in embeddings_span["attributes"] + or embeddings_span["attributes"]["gen_ai.request.model"] is None ) -def test_langchain_embeddings_integration_disabled(sentry_init, capture_events): +def test_langchain_embeddings_integration_disabled(sentry_init, capture_items): """Test that embeddings are not traced when integration is disabled.""" try: from langchain_openai import OpenAIEmbeddings @@ -1664,8 +1723,8 @@ def test_langchain_embeddings_integration_disabled(sentry_init, capture_events): pytest.skip("langchain_openai not installed") # Initialize without LangchainIntegration - sentry_init(traces_sample_rate=1.0) - events = capture_events() + sentry_init(traces_sample_rate=1.0, _experiments={"gen_ai_as_v2_spans": True}) + items = capture_items("transaction", "span") with mock.patch.object( OpenAIEmbeddings, @@ -1680,18 +1739,17 @@ def test_langchain_embeddings_integration_disabled(sentry_init, capture_events): embeddings.embed_documents(["Test"]) # Check that no embeddings spans were created - if events: - tx = events[0] - embeddings_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.embeddings" - ] - # Should be empty since integration is disabled - assert len(embeddings_spans) == 0 + spans = [item.payload for item in items if item.type == "span"] + embeddings_spans = [ + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" + ] + # Should be empty since integration is disabled + assert len(embeddings_spans) == 0 -def test_langchain_embeddings_multiple_providers(sentry_init, capture_events): +def test_langchain_embeddings_multiple_providers(sentry_init, capture_items): """Test that embeddings work with different providers.""" try: from langchain_openai import OpenAIEmbeddings, AzureOpenAIEmbeddings @@ -1703,7 +1761,7 @@ def test_langchain_embeddings_multiple_providers(sentry_init, capture_events): traces_sample_rate=1.0, send_default_pii=True, ) - events = capture_events() + items = capture_items("transaction", "span") # Mock both providers with mock.patch.object( @@ -1731,26 +1789,24 @@ def test_langchain_embeddings_multiple_providers(sentry_init, capture_events): openai_embeddings.embed_documents(["OpenAI test"]) azure_embeddings.embed_documents(["Azure test"]) - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] # Find embeddings spans embeddings_spans = [ - span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" ] # Should have 2 spans, one for each provider assert len(embeddings_spans) == 2 # Verify both spans have proper data for span in embeddings_spans: - assert span["data"]["gen_ai.operation.name"] == "embeddings" - assert span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["data"] + assert span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert span["attributes"]["gen_ai.request.model"] == "text-embedding-ada-002" + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] -def test_langchain_embeddings_error_handling(sentry_init, capture_events): +def test_langchain_embeddings_error_handling(sentry_init, capture_items): """Test that errors in embeddings are properly captured.""" try: from langchain_openai import OpenAIEmbeddings @@ -1762,7 +1818,7 @@ def test_langchain_embeddings_error_handling(sentry_init, capture_events): traces_sample_rate=1.0, send_default_pii=True, ) - events = capture_events() + items = capture_items("transaction", "span") # Mock the API call to raise an error with mock.patch.object( @@ -1781,15 +1837,16 @@ def test_langchain_embeddings_error_handling(sentry_init, capture_events): with pytest.raises(ValueError): embeddings.embed_documents(["Test"]) - # The error should be captured - assert len(events) >= 1 - # We should have both the transaction and potentially an error event - [e for e in events if e.get("level") == "error"] + [ + item.payload + for item in items + if item.type == "event" and item.payload.get("level") == "error" + ] # Note: errors might not be auto-captured depending on SDK settings, # but the span should still be created -def test_langchain_embeddings_multiple_calls(sentry_init, capture_events): +def test_langchain_embeddings_multiple_calls(sentry_init, capture_items): """Test that multiple embeddings calls within a transaction are all traced.""" try: from langchain_openai import OpenAIEmbeddings @@ -1801,7 +1858,7 @@ def test_langchain_embeddings_multiple_calls(sentry_init, capture_events): traces_sample_rate=1.0, send_default_pii=True, ) - events = capture_events() + items = capture_items("transaction", "span") # Mock the actual API calls with mock.patch.object( @@ -1828,32 +1885,31 @@ def test_langchain_embeddings_multiple_calls(sentry_init, capture_events): # Call embed_documents again embeddings.embed_documents(["Third batch"]) - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] # Find embeddings spans - should have 3 (2 embed_documents + 1 embed_query) embeddings_spans = [ - span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 3 # Verify all spans have proper data for span in embeddings_spans: - assert span["data"]["gen_ai.operation.name"] == "embeddings" - assert span["data"]["gen_ai.request.model"] == "text-embedding-ada-002" - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["data"] + assert span["attributes"]["gen_ai.operation.name"] == "embeddings" + assert span["attributes"]["gen_ai.request.model"] == "text-embedding-ada-002" + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] # Verify the input data is different for each span input_data_list = [ - span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] for span in embeddings_spans + span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + for span in embeddings_spans ] # They should all be different (different inputs) assert len(set(str(data) for data in input_data_list)) == 3 -def test_langchain_embeddings_span_hierarchy(sentry_init, capture_events): +def test_langchain_embeddings_span_hierarchy(sentry_init, capture_items): """Test that embeddings spans are properly nested within parent spans.""" try: from langchain_openai import OpenAIEmbeddings @@ -1865,7 +1921,7 @@ def test_langchain_embeddings_span_hierarchy(sentry_init, capture_events): traces_sample_rate=1.0, send_default_pii=True, ) - events = capture_events() + items = capture_items("transaction", "span") # Mock the actual API call with mock.patch.object( @@ -1884,15 +1940,15 @@ def test_langchain_embeddings_span_hierarchy(sentry_init, capture_events): with sentry_sdk.start_span(op="custom", name="custom operation"): embeddings.embed_documents(["Test within custom span"]) - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] # Find all spans embeddings_spans = [ - span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" ] + + tx = next(item.payload for item in items if item.type == "transaction") custom_spans = [span for span in tx.get("spans", []) if span.get("op") == "custom"] assert len(embeddings_spans) == 1 @@ -1902,11 +1958,11 @@ def test_langchain_embeddings_span_hierarchy(sentry_init, capture_events): embeddings_span = embeddings_spans[0] custom_span = custom_spans[0] - assert embeddings_span["data"]["gen_ai.operation.name"] == "embeddings" + assert embeddings_span["attributes"]["gen_ai.operation.name"] == "embeddings" assert custom_span["description"] == "custom operation" -def test_langchain_embeddings_with_list_and_string_inputs(sentry_init, capture_events): +def test_langchain_embeddings_with_list_and_string_inputs(sentry_init, capture_items): """Test that embeddings correctly handle both list and string inputs.""" try: from langchain_openai import OpenAIEmbeddings @@ -1918,7 +1974,7 @@ def test_langchain_embeddings_with_list_and_string_inputs(sentry_init, capture_e traces_sample_rate=1.0, send_default_pii=True, ) - events = capture_events() + items = capture_items("transaction", "span") # Mock the actual API calls with mock.patch.object( @@ -1943,21 +1999,19 @@ def test_langchain_embeddings_with_list_and_string_inputs(sentry_init, capture_e # embed_query takes a string embeddings.embed_query("Single string query") - # Check captured events - assert len(events) >= 1 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] # Find embeddings spans embeddings_spans = [ - span for span in tx.get("spans", []) if span.get("op") == "gen_ai.embeddings" + span + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.embeddings" ] assert len(embeddings_spans) == 2 # Both should have input data captured as lists for span in embeddings_spans: - assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["data"] - input_data = span["data"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] + assert SPANDATA.GEN_AI_EMBEDDINGS_INPUT in span["attributes"] + input_data = span["attributes"][SPANDATA.GEN_AI_EMBEDDINGS_INPUT] # Input should be normalized to list format if isinstance(input_data, str): # If serialized, should contain the input text @@ -1975,7 +2029,7 @@ def test_langchain_embeddings_with_list_and_string_inputs(sentry_init, capture_e ) def test_langchain_response_model_extraction( sentry_init, - capture_events, + capture_items, response_metadata_model, expected_model, ): @@ -1984,7 +2038,7 @@ def test_langchain_response_model_extraction( traces_sample_rate=1.0, send_default_pii=True, ) - events = capture_events() + items = capture_items("transaction", "span") callback = SentryLangchainCallback(max_span_map_size=100, include_prompts=True) @@ -2009,236 +2063,22 @@ def test_langchain_response_model_extraction( response = Mock(generations=[[generation]]) callback.on_llm_end(response=response, run_id=run_id) - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] llm_spans = [ span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.text_completion" + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.text_completion" ] assert len(llm_spans) > 0 llm_span = llm_spans[0] - assert llm_span["data"]["gen_ai.operation.name"] == "text_completion" + assert llm_span["attributes"]["gen_ai.operation.name"] == "text_completion" if expected_model is not None: - assert SPANDATA.GEN_AI_RESPONSE_MODEL in llm_span["data"] - assert llm_span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == expected_model + assert SPANDATA.GEN_AI_RESPONSE_MODEL in llm_span["attributes"] + assert llm_span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == expected_model else: - assert SPANDATA.GEN_AI_RESPONSE_MODEL not in llm_span.get("data", {}) - - -# Tests for multimodal content transformation functions - - -class TestTransformLangchainContentBlock: - """Tests for _transform_langchain_content_block function.""" - - def test_transform_image_base64(self): - """Test transformation of base64-encoded image content.""" - content_block = { - "type": "image", - "base64": "/9j/4AAQSkZJRgABAQAAAQABAAD...", - "mime_type": "image/jpeg", - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "blob", - "modality": "image", - "mime_type": "image/jpeg", - "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", - } - - def test_transform_image_url(self): - """Test transformation of URL-referenced image content.""" - content_block = { - "type": "image", - "url": "https://example.com/image.jpg", - "mime_type": "image/jpeg", - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "uri", - "modality": "image", - "mime_type": "image/jpeg", - "uri": "https://example.com/image.jpg", - } - - def test_transform_image_file_id(self): - """Test transformation of file_id-referenced image content.""" - content_block = { - "type": "image", - "file_id": "file-abc123", - "mime_type": "image/png", - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "file", - "modality": "image", - "mime_type": "image/png", - "file_id": "file-abc123", - } - - def test_transform_image_url_legacy_with_data_uri(self): - """Test transformation of legacy image_url format with data: URI (base64).""" - content_block = { - "type": "image_url", - "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD"}, - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "blob", - "modality": "image", - "mime_type": "image/jpeg", - "content": "/9j/4AAQSkZJRgABAQAAAQABAAD", - } - - def test_transform_image_url_legacy_with_http_url(self): - """Test transformation of legacy image_url format with HTTP URL.""" - content_block = { - "type": "image_url", - "image_url": {"url": "https://example.com/image.png"}, - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "uri", - "modality": "image", - "mime_type": "", - "uri": "https://example.com/image.png", - } - - def test_transform_image_url_legacy_string_url(self): - """Test transformation of legacy image_url format with string URL.""" - content_block = { - "type": "image_url", - "image_url": "https://example.com/image.gif", - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "uri", - "modality": "image", - "mime_type": "", - "uri": "https://example.com/image.gif", - } - - def test_transform_image_url_legacy_data_uri_png(self): - """Test transformation of legacy image_url format with PNG data URI.""" - content_block = { - "type": "image_url", - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" - }, - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "blob", - "modality": "image", - "mime_type": "image/png", - "content": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", - } - - def test_transform_missing_mime_type(self): - """Test transformation when mime_type is not provided.""" - content_block = { - "type": "image", - "base64": "/9j/4AAQSkZJRgABAQAAAQABAAD...", - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "blob", - "modality": "image", - "mime_type": "", - "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", - } - - def test_transform_anthropic_source_base64(self): - """Test transformation of Anthropic-style image with base64 source.""" - content_block = { - "type": "image", - "source": { - "type": "base64", - "media_type": "image/png", - "data": "iVBORw0KGgoAAAANSUhEUgAAAAE...", - }, - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "blob", - "modality": "image", - "mime_type": "image/png", - "content": "iVBORw0KGgoAAAANSUhEUgAAAAE...", - } - - def test_transform_anthropic_source_url(self): - """Test transformation of Anthropic-style image with URL source.""" - content_block = { - "type": "image", - "source": { - "type": "url", - "media_type": "image/jpeg", - "url": "https://example.com/image.jpg", - }, - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "uri", - "modality": "image", - "mime_type": "image/jpeg", - "uri": "https://example.com/image.jpg", - } - - def test_transform_anthropic_source_without_media_type(self): - """Test transformation of Anthropic-style image without media_type uses empty mime_type.""" - content_block = { - "type": "image", - "mime_type": "image/webp", # Top-level mime_type is ignored by standard Anthropic format - "source": { - "type": "base64", - "data": "UklGRh4AAABXRUJQVlA4IBIAAAAwAQCdASoBAAEAAQAcJYgCdAEO", - }, - } - result = _transform_langchain_content_block(content_block) - # Note: The shared transform_content_part uses media_type from source, not top-level mime_type - assert result == { - "type": "blob", - "modality": "image", - "mime_type": "", - "content": "UklGRh4AAABXRUJQVlA4IBIAAAAwAQCdASoBAAEAAQAcJYgCdAEO", - } - - def test_transform_google_inline_data(self): - """Test transformation of Google-style inline_data format.""" - content_block = { - "inline_data": { - "mime_type": "image/jpeg", - "data": "/9j/4AAQSkZJRgABAQAAAQABAAD...", - } - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "blob", - "modality": "image", - "mime_type": "image/jpeg", - "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", - } - - def test_transform_google_file_data(self): - """Test transformation of Google-style file_data format.""" - content_block = { - "file_data": { - "mime_type": "image/png", - "file_uri": "gs://bucket/path/to/image.png", - } - } - result = _transform_langchain_content_block(content_block) - assert result == { - "type": "uri", - "modality": "image", - "mime_type": "image/png", - "uri": "gs://bucket/path/to/image.png", - } + assert SPANDATA.GEN_AI_RESPONSE_MODEL not in llm_span.get("attributes", {}) @pytest.mark.parametrize( @@ -2286,13 +2126,13 @@ def test_transform_google_file_data(self): ], ) def test_langchain_ai_system_detection( - sentry_init, capture_events, ai_type, expected_system + sentry_init, capture_items, ai_type, expected_system ): sentry_init( integrations=[LangchainIntegration()], traces_sample_rate=1.0, ) - events = capture_events() + items = capture_items("transaction", "span") callback = SentryLangchainCallback(max_span_map_size=100, include_prompts=True) @@ -2312,136 +2152,17 @@ def test_langchain_ai_system_detection( response = Mock(generations=[[generation]]) callback.on_llm_end(response=response, run_id=run_id) - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - + spans = [item.payload for item in items if item.type == "span"] llm_spans = [ span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.text_completion" + for span in spans + if span["attributes"].get("sentry.op") == "gen_ai.text_completion" ] assert len(llm_spans) > 0 llm_span = llm_spans[0] if expected_system is not None: - assert llm_span["data"][SPANDATA.GEN_AI_SYSTEM] == expected_system + assert llm_span["attributes"][SPANDATA.GEN_AI_SYSTEM] == expected_system else: - assert SPANDATA.GEN_AI_SYSTEM not in llm_span.get("data", {}) - - -class TestTransformLangchainMessageContent: - """Tests for _transform_langchain_message_content function.""" - - def test_transform_string_content(self): - """Test that string content is returned unchanged.""" - result = _transform_langchain_message_content("Hello, world!") - assert result == "Hello, world!" - - def test_transform_list_with_text_blocks(self): - """Test transformation of list with text blocks (unchanged).""" - content = [ - {"type": "text", "text": "First message"}, - {"type": "text", "text": "Second message"}, - ] - result = _transform_langchain_message_content(content) - assert result == content - - def test_transform_list_with_image_blocks(self): - """Test transformation of list containing image blocks.""" - content = [ - {"type": "text", "text": "Check out this image:"}, - { - "type": "image", - "base64": "/9j/4AAQSkZJRgABAQAAAQABAAD...", - "mime_type": "image/jpeg", - }, - ] - result = _transform_langchain_message_content(content) - assert len(result) == 2 - assert result[0] == {"type": "text", "text": "Check out this image:"} - assert result[1] == { - "type": "blob", - "modality": "image", - "mime_type": "image/jpeg", - "content": "/9j/4AAQSkZJRgABAQAAAQABAAD...", - } - - def test_transform_list_with_mixed_content(self): - """Test transformation of list with mixed content types.""" - content = [ - {"type": "text", "text": "Here are some files:"}, - { - "type": "image", - "url": "https://example.com/image.jpg", - "mime_type": "image/jpeg", - }, - { - "type": "file", - "file_id": "doc-123", - "mime_type": "application/pdf", - }, - {"type": "audio", "base64": "audio_data...", "mime_type": "audio/mp3"}, - ] - result = _transform_langchain_message_content(content) - assert len(result) == 4 - assert result[0] == {"type": "text", "text": "Here are some files:"} - assert result[1] == { - "type": "uri", - "modality": "image", - "mime_type": "image/jpeg", - "uri": "https://example.com/image.jpg", - } - assert result[2] == { - "type": "file", - "modality": "document", - "mime_type": "application/pdf", - "file_id": "doc-123", - } - assert result[3] == { - "type": "blob", - "modality": "audio", - "mime_type": "audio/mp3", - "content": "audio_data...", - } - - def test_transform_list_with_non_dict_items(self): - """Test transformation handles non-dict items in list.""" - content = ["plain string", {"type": "text", "text": "dict text"}] - result = _transform_langchain_message_content(content) - assert result == ["plain string", {"type": "text", "text": "dict text"}] - - def test_transform_tuple_content(self): - """Test transformation of tuple content.""" - content = ( - {"type": "text", "text": "Message"}, - {"type": "image", "base64": "data...", "mime_type": "image/png"}, - ) - result = _transform_langchain_message_content(content) - assert len(result) == 2 - assert result[1] == { - "type": "blob", - "modality": "image", - "mime_type": "image/png", - "content": "data...", - } - - def test_transform_list_with_legacy_image_url(self): - """Test transformation of list containing legacy image_url blocks.""" - content = [ - {"type": "text", "text": "Check this:"}, - { - "type": "image_url", - "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ..."}, - }, - ] - result = _transform_langchain_message_content(content) - assert len(result) == 2 - assert result[0] == {"type": "text", "text": "Check this:"} - assert result[1] == { - "type": "blob", - "modality": "image", - "mime_type": "image/jpeg", - "content": "/9j/4AAQ...", - } + assert SPANDATA.GEN_AI_SYSTEM not in llm_span.get("attributes", {}) From 4815d2f23934c326acd056cdaae70f669cf0adaf Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 21 Apr 2026 09:08:23 +0200 Subject: [PATCH 5/5] fix test --- tests/integrations/langgraph/test_langgraph.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index 91fea2d760..86d6ee1f05 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -272,8 +272,9 @@ def original_invoke(self, *args, **kwargs): import json request_messages = json.loads(request_messages) - assert len(request_messages) == 1 - assert request_messages[0]["content"] == "Of course! How can I assist you?" + assert len(request_messages) == 2 + assert request_messages[0]["content"] == "Hello, can you help me?" + assert request_messages[1]["content"] == "Of course! How can I assist you?" response_text = invoke_span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] assert response_text == expected_assistant_response