diff --git a/.github/workflows/test-integrations-ai.yml b/.github/workflows/test-integrations-ai.yml index 99f59df833..5b31a20a35 100644 --- a/.github/workflows/test-integrations-ai.yml +++ b/.github/workflows/test-integrations-ai.yml @@ -59,6 +59,10 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-openai-latest" + - name: Test openai_agents latest + run: | + set -x # print commands that are executed + ./scripts/runtox.sh "py${{ matrix.python-version }}-openai_agents-latest" - name: Test huggingface_hub latest run: | set -x # print commands that are executed @@ -121,6 +125,10 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai" + - name: Test openai_agents pinned + run: | + set -x # print commands that are executed + ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai_agents" - name: Test huggingface_hub pinned run: | set -x # print commands that are executed diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index 62e916e5bd..7e48d620c0 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -152,6 +152,12 @@ "loguru": { "package": "loguru", }, + "openai_agents": { + "package": "openai-agents", + "deps": { + "*": ["pytest-asyncio"], + }, + }, "openfeature": { "package": "openfeature-sdk", }, diff --git a/scripts/populate_tox/tox.jinja b/scripts/populate_tox/tox.jinja index c3f76f99f6..66b1d7885a 100644 --- a/scripts/populate_tox/tox.jinja +++ b/scripts/populate_tox/tox.jinja @@ -372,6 +372,7 @@ setenv = litestar: TESTPATH=tests/integrations/litestar loguru: TESTPATH=tests/integrations/loguru openai: TESTPATH=tests/integrations/openai + openai_agents: TESTPATH=tests/integrations/openai_agents openfeature: TESTPATH=tests/integrations/openfeature pure_eval: TESTPATH=tests/integrations/pure_eval pymongo: TESTPATH=tests/integrations/pymongo diff --git a/scripts/split_tox_gh_actions/split_tox_gh_actions.py b/scripts/split_tox_gh_actions/split_tox_gh_actions.py index 3fbc0ec1c5..af1ff84cd6 100755 --- a/scripts/split_tox_gh_actions/split_tox_gh_actions.py +++ b/scripts/split_tox_gh_actions/split_tox_gh_actions.py @@ -63,6 +63,7 @@ "cohere", "langchain", "openai", + "openai_agents", "huggingface_hub", ], "Cloud": [ diff --git a/sentry_sdk/integrations/__init__.py b/sentry_sdk/integrations/__init__.py index 5485ebe4c3..3a4804985a 100644 --- a/sentry_sdk/integrations/__init__.py +++ b/sentry_sdk/integrations/__init__.py @@ -147,6 +147,7 @@ def iter_default_integrations( "launchdarkly": (9, 8, 0), "loguru": (0, 7, 0), "openai": (1, 0, 0), + "openai_agents": (0, 0, 19), "openfeature": (0, 7, 1), "quart": (0, 16, 0), "ray": (2, 7, 0), diff --git a/sentry_sdk/integrations/openai_agents/__init__.py b/sentry_sdk/integrations/openai_agents/__init__.py new file mode 100644 index 0000000000..384cf9c651 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/__init__.py @@ -0,0 +1,49 @@ +from sentry_sdk.integrations import DidNotEnable, Integration + +from .patches import ( + _create_get_model_wrapper, + _create_get_all_tools_wrapper, + _create_run_wrapper, + _patch_agent_run, +) + +try: + import agents + +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +def _patch_runner() -> None: + # Create the root span for one full agent run (including eventual handoffs) + # Note agents.run.DEFAULT_AGENT_RUNNER.run_sync is a wrapper around + # agents.run.DEFAULT_AGENT_RUNNER.run. It does not need to be wrapped separately. + # TODO-anton: Also patch streaming runner: agents.Runner.run_streamed + agents.run.DEFAULT_AGENT_RUNNER.run = _create_run_wrapper( + agents.run.DEFAULT_AGENT_RUNNER.run + ) + + # Creating the actual spans for each agent run. + _patch_agent_run() + + +def _patch_model() -> None: + agents.run.AgentRunner._get_model = classmethod( + _create_get_model_wrapper(agents.run.AgentRunner._get_model), + ) + + +def _patch_tools() -> None: + agents.run.AgentRunner._get_all_tools = classmethod( + _create_get_all_tools_wrapper(agents.run.AgentRunner._get_all_tools), + ) + + +class OpenAIAgentsIntegration(Integration): + identifier = "openai_agents" + + @staticmethod + def setup_once() -> None: + _patch_tools() + _patch_model() + _patch_runner() diff --git a/sentry_sdk/integrations/openai_agents/consts.py b/sentry_sdk/integrations/openai_agents/consts.py new file mode 100644 index 0000000000..f5de978be0 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/consts.py @@ -0,0 +1 @@ +SPAN_ORIGIN = "auto.ai.openai_agents" diff --git a/sentry_sdk/integrations/openai_agents/patches/__init__.py b/sentry_sdk/integrations/openai_agents/patches/__init__.py new file mode 100644 index 0000000000..06bb1711f8 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/patches/__init__.py @@ -0,0 +1,4 @@ +from .models import _create_get_model_wrapper # noqa: F401 +from .tools import _create_get_all_tools_wrapper # noqa: F401 +from .runner import _create_run_wrapper # noqa: F401 +from .agent_run import _patch_agent_run # noqa: F401 diff --git a/sentry_sdk/integrations/openai_agents/patches/agent_run.py b/sentry_sdk/integrations/openai_agents/patches/agent_run.py new file mode 100644 index 0000000000..984be12367 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/patches/agent_run.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from functools import wraps + +from sentry_sdk.integrations import DidNotEnable + +from ..spans import invoke_agent_span, update_invoke_agent_span, handoff_span + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Optional + + +try: + import agents +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +def _patch_agent_run() -> None: + """ + Patches AgentRunner methods to create agent invocation spans. + This directly patches the execution flow to track when agents start and stop. + """ + + # Store original methods + original_run_single_turn = agents.run.AgentRunner._run_single_turn + original_execute_handoffs = agents._run_impl.RunImpl.execute_handoffs + original_execute_final_output = agents._run_impl.RunImpl.execute_final_output + + def _start_invoke_agent_span( + context_wrapper: agents.RunContextWrapper, agent: agents.Agent + ) -> None: + """Start an agent invocation span""" + # Store the agent on the context wrapper so we can access it later + context_wrapper._sentry_current_agent = agent + invoke_agent_span(context_wrapper, agent) + + def _end_invoke_agent_span( + context_wrapper: agents.RunContextWrapper, + agent: agents.Agent, + output: Optional[Any] = None, + ) -> None: + """End the agent invocation span""" + # Clear the stored agent + if hasattr(context_wrapper, "_sentry_current_agent"): + delattr(context_wrapper, "_sentry_current_agent") + + update_invoke_agent_span(context_wrapper, agent, output) + + def _has_active_agent_span(context_wrapper: agents.RunContextWrapper) -> bool: + """Check if there's an active agent span for this context""" + return getattr(context_wrapper, "_sentry_current_agent", None) is not None + + def _get_current_agent( + context_wrapper: agents.RunContextWrapper, + ) -> Optional[agents.Agent]: + """Get the current agent from context wrapper""" + return getattr(context_wrapper, "_sentry_current_agent", None) + + @wraps( + original_run_single_turn.__func__ + if hasattr(original_run_single_turn, "__func__") + else original_run_single_turn + ) + async def patched_run_single_turn( + cls: agents.Runner, *args: Any, **kwargs: Any + ) -> Any: + """Patched _run_single_turn that creates agent invocation spans""" + agent = kwargs.get("agent") + context_wrapper = kwargs.get("context_wrapper") + should_run_agent_start_hooks = kwargs.get("should_run_agent_start_hooks") + + # Start agent span when agent starts (but only once per agent) + if should_run_agent_start_hooks and agent and context_wrapper: + # End any existing span for a different agent + if _has_active_agent_span(context_wrapper): + current_agent = _get_current_agent(context_wrapper) + if current_agent and current_agent != agent: + _end_invoke_agent_span(context_wrapper, current_agent) + + _start_invoke_agent_span(context_wrapper, agent) + + # Call original method with all the correct parameters + try: + result = await original_run_single_turn(*args, **kwargs) + finally: + if agent and context_wrapper and _has_active_agent_span(context_wrapper): + _end_invoke_agent_span(context_wrapper, agent) + + return result + + @wraps( + original_execute_handoffs.__func__ + if hasattr(original_execute_handoffs, "__func__") + else original_execute_handoffs + ) + async def patched_execute_handoffs( + cls: agents.Runner, *args: Any, **kwargs: Any + ) -> Any: + """Patched execute_handoffs that creates handoff spans and ends agent span for handoffs""" + context_wrapper = kwargs.get("context_wrapper") + run_handoffs = kwargs.get("run_handoffs") + agent = kwargs.get("agent") + + # Create Sentry handoff span for the first handoff (agents library only processes the first one) + if run_handoffs: + first_handoff = run_handoffs[0] + handoff_agent_name = first_handoff.handoff.agent_name + handoff_span(context_wrapper, agent, handoff_agent_name) + + # Call original method with all parameters + try: + result = await original_execute_handoffs(*args, **kwargs) + + finally: + # End span for current agent after handoff processing is complete + if agent and context_wrapper and _has_active_agent_span(context_wrapper): + _end_invoke_agent_span(context_wrapper, agent) + + return result + + @wraps( + original_execute_final_output.__func__ + if hasattr(original_execute_final_output, "__func__") + else original_execute_final_output + ) + async def patched_execute_final_output( + cls: agents.Runner, *args: Any, **kwargs: Any + ) -> Any: + """Patched execute_final_output that ends agent span for final outputs""" + agent = kwargs.get("agent") + context_wrapper = kwargs.get("context_wrapper") + final_output = kwargs.get("final_output") + + # Call original method with all parameters + try: + result = await original_execute_final_output(*args, **kwargs) + finally: + # End span for current agent after final output processing is complete + if agent and context_wrapper and _has_active_agent_span(context_wrapper): + _end_invoke_agent_span(context_wrapper, agent, final_output) + + return result + + # Apply patches + agents.run.AgentRunner._run_single_turn = classmethod(patched_run_single_turn) + agents._run_impl.RunImpl.execute_handoffs = classmethod(patched_execute_handoffs) + agents._run_impl.RunImpl.execute_final_output = classmethod( + patched_execute_final_output + ) diff --git a/sentry_sdk/integrations/openai_agents/patches/models.py b/sentry_sdk/integrations/openai_agents/patches/models.py new file mode 100644 index 0000000000..e1dbb58719 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/patches/models.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from functools import wraps + +from sentry_sdk.integrations import DidNotEnable + +from ..spans import ai_client_span, update_ai_client_span + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Callable + + +try: + import agents +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +def _create_get_model_wrapper( + original_get_model: Callable[..., Any], +) -> Callable[..., Any]: + """ + Wraps the agents.Runner._get_model method to wrap the get_response method of the model to create a AI client span. + """ + + @wraps( + original_get_model.__func__ + if hasattr(original_get_model, "__func__") + else original_get_model + ) + def wrapped_get_model( + cls: agents.Runner, agent: agents.Agent, run_config: agents.RunConfig + ) -> agents.Model: + model = original_get_model(agent, run_config) + original_get_response = model.get_response + + @wraps(original_get_response) + async def wrapped_get_response(*args: Any, **kwargs: Any) -> Any: + with ai_client_span(agent, kwargs) as span: + result = await original_get_response(*args, **kwargs) + + update_ai_client_span(span, agent, kwargs, result) + + return result + + model.get_response = wrapped_get_response + + return model + + return wrapped_get_model diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py new file mode 100644 index 0000000000..22d5c4afff --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from functools import wraps + +import sentry_sdk + +from ..spans import agent_workflow_span +from ..utils import _capture_exception + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Callable + + +def _create_run_wrapper(original_func: Callable[..., Any]) -> Callable[..., Any]: + """ + Wraps the agents.Runner.run methods to create a root span for the agent workflow runs. + + Note agents.Runner.run_sync() is a wrapper around agents.Runner.run(), + so it does not need to be wrapped separately. + """ + + @wraps(original_func) + async def wrapper(*args: Any, **kwargs: Any) -> Any: + agent = args[0] + with agent_workflow_span(agent): + result = None + try: + result = await original_func(*args, **kwargs) + return result + except Exception as exc: + _capture_exception(exc) + + # It could be that there is a "invoke agent" span still open + span = sentry_sdk.get_current_span() + if span is not None and span.timestamp is None: + span.__exit__(None, None, None) + + raise exc from None + + return wrapper diff --git a/sentry_sdk/integrations/openai_agents/patches/tools.py b/sentry_sdk/integrations/openai_agents/patches/tools.py new file mode 100644 index 0000000000..ba75cd0eb8 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/patches/tools.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from functools import wraps + +from sentry_sdk.integrations import DidNotEnable + +from ..spans import execute_tool_span, update_execute_tool_span + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Callable + +try: + import agents +except ImportError: + raise DidNotEnable("OpenAI Agents not installed") + + +def _create_get_all_tools_wrapper( + original_get_all_tools: Callable[..., Any], +) -> Callable[..., Any]: + """ + Wraps the agents.Runner._get_all_tools method of the Runner class to wrap all function tools with Sentry instrumentation. + """ + + @wraps( + original_get_all_tools.__func__ + if hasattr(original_get_all_tools, "__func__") + else original_get_all_tools + ) + async def wrapped_get_all_tools( + cls: agents.Runner, + agent: agents.Agent, + context_wrapper: agents.RunContextWrapper, + ) -> list[agents.Tool]: + # Get the original tools + tools = await original_get_all_tools(agent, context_wrapper) + + wrapped_tools = [] + for tool in tools: + # Wrap only the function tools (for now) + if tool.__class__.__name__ != "FunctionTool": + wrapped_tools.append(tool) + continue + + # Create a new FunctionTool with our wrapped invoke method + original_on_invoke = tool.on_invoke_tool + + def create_wrapped_invoke( + current_tool: agents.Tool, current_on_invoke: Callable[..., Any] + ) -> Callable[..., Any]: + @wraps(current_on_invoke) + async def sentry_wrapped_on_invoke_tool( + *args: Any, **kwargs: Any + ) -> Any: + with execute_tool_span(current_tool, *args, **kwargs) as span: + # We can not capture exceptions in tool execution here because + # `_on_invoke_tool` is swallowing the exception here: + # https://github.com/openai/openai-agents-python/blob/main/src/agents/tool.py#L409-L422 + # And because function_tool is a decorator with `default_tool_error_function` set as a default parameter + # I was unable to monkey patch it because those are evaluated at module import time + # and the SDK is too late to patch it. I was also unable to patch `_on_invoke_tool_impl` + # because it is nested inside this import time code. As if they made it hard to patch on purpose... + result = await current_on_invoke(*args, **kwargs) + update_execute_tool_span(span, agent, current_tool, result) + + return result + + return sentry_wrapped_on_invoke_tool + + wrapped_tool = agents.FunctionTool( + name=tool.name, + description=tool.description, + params_json_schema=tool.params_json_schema, + on_invoke_tool=create_wrapped_invoke(tool, original_on_invoke), + strict_json_schema=tool.strict_json_schema, + is_enabled=tool.is_enabled, + ) + wrapped_tools.append(wrapped_tool) + + return wrapped_tools + + return wrapped_get_all_tools diff --git a/sentry_sdk/integrations/openai_agents/spans/__init__.py b/sentry_sdk/integrations/openai_agents/spans/__init__.py new file mode 100644 index 0000000000..3bc453cafa --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/spans/__init__.py @@ -0,0 +1,5 @@ +from .agent_workflow import agent_workflow_span # noqa: F401 +from .ai_client import ai_client_span, update_ai_client_span # noqa: F401 +from .execute_tool import execute_tool_span, update_execute_tool_span # noqa: F401 +from .handoff import handoff_span # noqa: F401 +from .invoke_agent import invoke_agent_span, update_invoke_agent_span # noqa: F401 diff --git a/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py new file mode 100644 index 0000000000..4d12fff782 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/spans/agent_workflow.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import sentry_sdk + +from ..consts import SPAN_ORIGIN + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import agents + + +def agent_workflow_span(agent: agents.Agent) -> sentry_sdk.tracing.Span: + # Create a transaction or a span if an transaction is already active + span = sentry_sdk.start_span( + name=f"{agent.name} workflow", + origin=SPAN_ORIGIN, + ) + + return span diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index d325ae86e3..a67d3add5c 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import sentry_sdk from sentry_sdk.consts import OP, SPANDATA @@ -16,23 +18,28 @@ from typing import Any -def ai_client_span(agent, get_response_kwargs): - # type: (Agent, dict[str, Any]) -> sentry_sdk.tracing.Span - # TODO-anton: implement other types of operations. Now "chat" is hardcoded. +def ai_client_span( + agent: Agent, get_response_kwargs: dict[str, Any] +) -> sentry_sdk.tracing.Span: model_name = agent.model.model if hasattr(agent.model, "model") else agent.model + # TODO-anton: implement other types of operations. Now "chat" is hardcoded. span = sentry_sdk.start_span( op=OP.GEN_AI_CHAT, description=f"chat {model_name}", origin=SPAN_ORIGIN, ) # TODO-anton: remove hardcoded stuff and replace something that also works for embedding and so on - span.set_data(SPANDATA.GEN_AI_OPERATION_NAME, "chat") + span.set_attribute(SPANDATA.GEN_AI_OPERATION_NAME, "chat") return span -def update_ai_client_span(span, agent, get_response_kwargs, result): - # type: (sentry_sdk.tracing.Span, Agent, dict[str, Any], Any) -> None +def update_ai_client_span( + span: sentry_sdk.tracing.Span, + agent: Agent, + get_response_kwargs: dict[str, Any], + result: Any, +) -> None: _set_agent_data(span, agent) _set_usage_data(span, result.usage) _set_input_data(span, get_response_kwargs) diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py new file mode 100644 index 0000000000..af130a137e --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import sentry_sdk +from sentry_sdk.consts import OP, SPANDATA +from sentry_sdk.scope import should_send_default_pii + +from ..consts import SPAN_ORIGIN +from ..utils import _set_agent_data + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import agents + from typing import Any + + +def execute_tool_span( + tool: agents.Tool, *args: Any, **kwargs: Any +) -> sentry_sdk.tracing.Span: + span = sentry_sdk.start_span( + op=OP.GEN_AI_EXECUTE_TOOL, + name=f"execute_tool {tool.name}", + origin=SPAN_ORIGIN, + ) + + span.set_attribute(SPANDATA.GEN_AI_OPERATION_NAME, "execute_tool") + + if tool.__class__.__name__ == "FunctionTool": + span.set_attribute(SPANDATA.GEN_AI_TOOL_TYPE, "function") + + span.set_attribute(SPANDATA.GEN_AI_TOOL_NAME, tool.name) + span.set_attribute(SPANDATA.GEN_AI_TOOL_DESCRIPTION, tool.description) + + if should_send_default_pii(): + input = args[1] + span.set_attribute(SPANDATA.GEN_AI_TOOL_INPUT, input) + + return span + + +def update_execute_tool_span( + span: sentry_sdk.tracing.Span, agent: agents.Agent, tool: agents.Tool, result: Any +) -> None: + _set_agent_data(span, agent) + + if should_send_default_pii(): + span.set_attribute(SPANDATA.GEN_AI_TOOL_OUTPUT, result) diff --git a/sentry_sdk/integrations/openai_agents/spans/handoff.py b/sentry_sdk/integrations/openai_agents/spans/handoff.py new file mode 100644 index 0000000000..daa6579537 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/spans/handoff.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import sentry_sdk +from sentry_sdk.consts import OP, SPANDATA + +from ..consts import SPAN_ORIGIN + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import agents + + +def handoff_span( + context_wrapper: agents.RunContextWrapper, + from_agent: agents.Agent, + to_agent_name: str, +) -> None: + with sentry_sdk.start_span( + op=OP.GEN_AI_HANDOFF, + name=f"handoff from {from_agent.name} to {to_agent_name}", + origin=SPAN_ORIGIN, + ) as span: + span.set_attribute(SPANDATA.GEN_AI_OPERATION_NAME, "handoff") diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py new file mode 100644 index 0000000000..b4f9043616 --- /dev/null +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import sentry_sdk +from sentry_sdk.consts import OP, SPANDATA + +from ..consts import SPAN_ORIGIN +from ..utils import _set_agent_data + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import agents + from typing import Any + + +def invoke_agent_span( + context_wrapper: agents.RunContextWrapper, agent: agents.Agent +) -> sentry_sdk.tracing.Span: + span = sentry_sdk.start_span( + op=OP.GEN_AI_INVOKE_AGENT, + name=f"invoke_agent {agent.name}", + origin=SPAN_ORIGIN, + ) + span.__enter__() + + span.set_attribute(SPANDATA.GEN_AI_OPERATION_NAME, "invoke_agent") + + _set_agent_data(span, agent) + + context_wrapper._sentry_invoke_agent_span = span + + return span + + +def update_invoke_agent_span( + context_wrapper: agents.RunContextWrapper, agent: agents.Agent, output: Any +) -> None: + span = getattr(context_wrapper, "_sentry_invoke_agent_span", None) + if span is not None: + span.__exit__(None, None, None) + del context_wrapper._sentry_invoke_agent_span diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index dc66521c83..dd038993ae 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import sentry_sdk from sentry_sdk.consts import SPANDATA @@ -9,7 +11,6 @@ if TYPE_CHECKING: from typing import Any - from typing import Callable from typing import Union from agents import Usage @@ -20,8 +21,7 @@ raise DidNotEnable("OpenAI Agents not installed") -def _capture_exception(exc): - # type: (Any) -> None +def _capture_exception(exc: Any) -> None: event, hint = event_from_exception( exc, client_options=sentry_sdk.get_client().options, @@ -30,85 +30,75 @@ def _capture_exception(exc): sentry_sdk.capture_event(event, hint=hint) -def _get_start_span_function(): - # type: () -> Callable[..., Any] - current_span = sentry_sdk.get_current_span() - transaction_exists = ( - current_span is not None and current_span.containing_transaction == current_span - ) - return sentry_sdk.start_span if transaction_exists else sentry_sdk.start_transaction - - -def _set_agent_data(span, agent): - # type: (sentry_sdk.tracing.Span, agents.Agent) -> None - span.set_data( +def _set_agent_data(span: sentry_sdk.tracing.Span, agent: agents.Agent) -> None: + span.set_attribute( SPANDATA.GEN_AI_SYSTEM, "openai" ) # See footnote for https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/#gen-ai-system for explanation why. - span.set_data(SPANDATA.GEN_AI_AGENT_NAME, agent.name) + span.set_attribute(SPANDATA.GEN_AI_AGENT_NAME, agent.name) if agent.model_settings.max_tokens: - span.set_data( + span.set_attribute( SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, agent.model_settings.max_tokens ) if agent.model: model_name = agent.model.model if hasattr(agent.model, "model") else agent.model - span.set_data(SPANDATA.GEN_AI_REQUEST_MODEL, model_name) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model_name) if agent.model_settings.presence_penalty: - span.set_data( + span.set_attribute( SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, agent.model_settings.presence_penalty, ) if agent.model_settings.temperature: - span.set_data( + span.set_attribute( SPANDATA.GEN_AI_REQUEST_TEMPERATURE, agent.model_settings.temperature ) if agent.model_settings.top_p: - span.set_data(SPANDATA.GEN_AI_REQUEST_TOP_P, agent.model_settings.top_p) + span.set_attribute(SPANDATA.GEN_AI_REQUEST_TOP_P, agent.model_settings.top_p) if agent.model_settings.frequency_penalty: - span.set_data( + span.set_attribute( SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, agent.model_settings.frequency_penalty, ) if len(agent.tools) > 0: - span.set_data( + span.set_attribute( SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize([vars(tool) for tool in agent.tools]), ) -def _set_usage_data(span, usage): - # type: (sentry_sdk.tracing.Span, Usage) -> None - span.set_data(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, usage.input_tokens) - span.set_data( +def _set_usage_data(span: sentry_sdk.tracing.Span, usage: Usage) -> None: + span.set_attribute(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, usage.input_tokens) + span.set_attribute( SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, usage.input_tokens_details.cached_tokens, ) - span.set_data(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS, usage.output_tokens) - span.set_data( + span.set_attribute(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS, usage.output_tokens) + span.set_attribute( SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, usage.output_tokens_details.reasoning_tokens, ) - span.set_data(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, usage.total_tokens) + span.set_attribute(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, usage.total_tokens) -def _set_input_data(span, get_response_kwargs): - # type: (sentry_sdk.tracing.Span, dict[str, Any]) -> None +def _set_input_data( + span: sentry_sdk.tracing.Span, get_response_kwargs: dict[str, Any] +) -> None: if not should_send_default_pii(): return - messages_by_role = { + messages_by_role: dict[str, list[Any]] = { "system": [], "user": [], "assistant": [], "tool": [], - } # type: (dict[str, list[Any]]) + } system_instructions = get_response_kwargs.get("system_instructions") if system_instructions: messages_by_role["system"].append({"type": "text", "text": system_instructions}) @@ -129,22 +119,23 @@ def _set_input_data(span, get_response_kwargs): if len(messages) > 0: request_messages.append({"role": role, "content": messages}) - span.set_data(SPANDATA.GEN_AI_REQUEST_MESSAGES, safe_serialize(request_messages)) + span.set_attribute( + SPANDATA.GEN_AI_REQUEST_MESSAGES, safe_serialize(request_messages) + ) -def _set_output_data(span, result): - # type: (sentry_sdk.tracing.Span, Any) -> None +def _set_output_data(span: sentry_sdk.tracing.Span, result: Any) -> None: if not should_send_default_pii(): return - output_messages = { + output_messages: dict[str, list[Any]] = { "response": [], "tool": [], - } # type: (dict[str, list[Any]]) + } for output in result.output: if output.type == "function_call": - output_messages["tool"].append(output.dict()) + output_messages["tool"].append(output.model_dump()) elif output.type == "message": for output_message in output.content: try: @@ -154,22 +145,22 @@ def _set_output_data(span, result): output_messages["response"].append(output_message.dict()) if len(output_messages["tool"]) > 0: - span.set_data( + span.set_attribute( SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, safe_serialize(output_messages["tool"]) ) if len(output_messages["response"]) > 0: - span.set_data( + span.set_attribute( SPANDATA.GEN_AI_RESPONSE_TEXT, safe_serialize(output_messages["response"]) ) -def safe_serialize(data): - # type: (Any) -> str +def safe_serialize(data: Any) -> str: """Safely serialize to a readable string.""" - def serialize_item(item): - # type: (Any) -> Union[str, dict[Any, Any], list[Any], tuple[Any, ...]] + def serialize_item( + item: Any, + ) -> Union[str, dict[Any, Any], list[Any], tuple[Any, ...]]: if callable(item): try: module = getattr(item, "__module__", None) diff --git a/tests/integrations/openai_agents/__init__.py b/tests/integrations/openai_agents/__init__.py new file mode 100644 index 0000000000..6940e2bbbe --- /dev/null +++ b/tests/integrations/openai_agents/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.importorskip("agents") diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 37a066aeca..43a3396ebf 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -317,10 +317,10 @@ async def test_handoff_span(sentry_init, capture_events, mock_usage): (transaction,) = events spans = transaction["spans"] - handoff_span = spans[2] - # Verify handoff span was created - assert handoff_span is not None + # There should be exactly one handoff span + (handoff_span,) = [span for span in spans if span["op"] == "gen_ai.handoff"] + assert ( handoff_span["description"] == "handoff from primary_agent to secondary_agent" ) @@ -408,12 +408,25 @@ def simple_test_tool(message: str) -> str: (transaction,) = events spans = transaction["spans"] - ( - agent_span, - ai_client_span1, - tool_span, - ai_client_span2, - ) = spans + + assert len(spans) == 4 + + # Find each span by its characteristics + agent_span = next(s for s in spans if s["description"] == "invoke_agent test_agent") + tool_span = next( + s for s in spans if s["description"] == "execute_tool simple_test_tool" + ) + ai_client_span1 = next( + s + for s in spans + if s["description"] == "chat gpt-4" + and "gen_ai.response.tool_calls" in s["data"] + ) + ai_client_span2 = next( + s + for s in spans + if s["description"] == "chat gpt-4" and "gen_ai.response.text" in s["data"] + ) available_tools = safe_serialize( [ @@ -631,4 +644,4 @@ async def test_error_handling(sentry_init, capture_events, test_agent): assert ai_client_span["description"] == "chat gpt-4" assert ai_client_span["origin"] == "auto.ai.openai_agents" - assert ai_client_span["tags"]["status"] == "internal_error" + assert ai_client_span["status"] == "internal_error" diff --git a/tox.ini b/tox.ini index 83d2631aef..fb939d309e 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ # The file (and all resulting CI YAMLs) then need to be regenerated via # "scripts/generate-test-files.sh". # -# Last generated: 2025-06-25T13:33:01.471693+00:00 +# Last generated: 2025-07-08T14:00:07.609918+00:00 [tox] requires = @@ -141,6 +141,9 @@ envlist = {py3.9,py3.11,py3.12}-cohere-v5.11.4 {py3.9,py3.11,py3.12}-cohere-v5.15.0 + {py3.9,py3.11,py3.12}-openai_agents-v0.0.19 + {py3.9,py3.12,py3.13}-openai_agents-v0.1.0 + {py3.8,py3.10,py3.11}-huggingface_hub-v0.22.2 {py3.8,py3.11,py3.12}-huggingface_hub-v0.26.5 {py3.8,py3.12,py3.13}-huggingface_hub-v0.30.2 @@ -502,6 +505,10 @@ deps = cohere-v5.11.4: cohere==5.11.4 cohere-v5.15.0: cohere==5.15.0 + openai_agents-v0.0.19: openai-agents==0.0.19 + openai_agents-v0.1.0: openai-agents==0.1.0 + openai_agents: pytest-asyncio + huggingface_hub-v0.22.2: huggingface_hub==0.22.2 huggingface_hub-v0.26.5: huggingface_hub==0.26.5 huggingface_hub-v0.30.2: huggingface_hub==0.30.2 @@ -793,6 +800,7 @@ setenv = litestar: TESTPATH=tests/integrations/litestar loguru: TESTPATH=tests/integrations/loguru openai: TESTPATH=tests/integrations/openai + openai_agents: TESTPATH=tests/integrations/openai_agents openfeature: TESTPATH=tests/integrations/openfeature pure_eval: TESTPATH=tests/integrations/pure_eval pymongo: TESTPATH=tests/integrations/pymongo