From 2a43fbe7a901135c27ab6989bf21e7821e53d179 Mon Sep 17 00:00:00 2001 From: pradystar Date: Tue, 21 Jul 2026 11:57:40 -0700 Subject: [PATCH 01/10] feat(otel): normalize native and user-wired paths --- splunk-ao-a2a/README.md | 37 +- splunk-ao-a2a/examples/.env.example | 8 + .../tests/test_splunk_ao_compatibility.py | 80 ++++ src/splunk_ao/otel.py | 345 +++++++++------- tests/test_otel.py | 358 +++------------- tests/test_otel_native_paths.py | 387 ++++++++++++++++++ 6 files changed, 768 insertions(+), 447 deletions(-) create mode 100644 splunk-ao-a2a/tests/test_splunk_ao_compatibility.py create mode 100644 tests/test_otel_native_paths.py diff --git a/splunk-ao-a2a/README.md b/splunk-ao-a2a/README.md index 60453fb9..6904e3d1 100644 --- a/splunk-ao-a2a/README.md +++ b/splunk-ao-a2a/README.md @@ -39,18 +39,23 @@ Splunk AO observability for [A2A (Agent-to-Agent)](https://github.com/google/A2A pip install splunk-ao-a2a ``` -**Requirements:** Python 3.11+, a [Splunk AO API key](https://www.splunk.com/), and [a2a-sdk](https://pypi.org/project/a2a-sdk/) 0.3+ +**Requirements:** Python 3.11+, Splunk AO standalone or Splunk Observability Cloud credentials, and [a2a-sdk](https://pypi.org/project/a2a-sdk/) 0.3+ ## Quick Start ```python -from splunk_ao.otel import SplunkAOSpanProcessor, add_splunk_ao_span_processor +from splunk_ao.otel import add_splunk_ao_span_processor from splunk_ao_a2a import A2AInstrumentor from opentelemetry.sdk.trace import TracerProvider provider = TracerProvider() -add_splunk_ao_span_processor(provider, SplunkAOSpanProcessor()) +add_splunk_ao_span_processor(provider) A2AInstrumentor().instrument(tracer_provider=provider, agent_name="orchestrator") + +try: + run_application() +finally: + provider.shutdown() ``` Once instrumented, all `a2a-sdk` client and server interactions produce OTel spans automatically. @@ -63,14 +68,28 @@ Once instrumented, all `a2a-sdk` client and server interactions produce OTel spa | `agent_name` | Name of this agent, set on spans as `gen_ai.agent.name`. | | `capture_content` | Set to `False` to disable capturing message content (e.g. for PII compliance). | -Environment variables for the Splunk AO exporter: +For standalone Splunk AO: | Environment Variable | Description | |---------------------|-------------| | `SPLUNK_AO_API_KEY` | Splunk AO API key (required) | -| `SPLUNK_AO_CONSOLE_URL` | Splunk AO console URL (required for self-hosted deployments, e.g. `http://localhost:8088`) | -| `SPLUNK_AO_PROJECT` | Project name (alternative to `SplunkAOSpanProcessor(project=...)`) | -| `SPLUNK_AO_LOG_STREAM` | Log stream name (alternative to `SplunkAOSpanProcessor(logstream=...)`) | +| `SPLUNK_AO_CONSOLE_URL` | Splunk AO console URL (required) | +| `SPLUNK_AO_API_URL` | Explicit API URL (optional; otherwise derived from the console URL) | +| `SPLUNK_AO_PROJECT` / `SPLUNK_AO_PROJECT_ID` | Project name or ID | +| `SPLUNK_AO_LOG_STREAM` / `SPLUNK_AO_LOG_STREAM_ID` | Log-stream name or ID | + +For Splunk Observability Cloud: + +| Environment Variable | Description | +|---------------------|-------------| +| `SPLUNK_AO_REALM` | Observability Cloud realm (required) | +| `SPLUNK_AO_SF_TOKEN` | SignalFlow ingest token used for OTLP export (required) | +| `SPLUNK_AO_PROJECT` / `SPLUNK_AO_PROJECT_ID` | Optional project routing | +| `SPLUNK_AO_LOG_STREAM` / `SPLUNK_AO_LOG_STREAM_ID` | Optional log-stream routing | + +O11y routing may be omitted entirely. In that case, the exporter sends only +authentication and ingestion assigns the trace to the unknown-project bucket. +The same Python setup works for both deployments; only the environment changes. ## Features @@ -120,7 +139,7 @@ from a2a.types import ( AgentCapabilities, AgentCard, AgentSkill, Message, Role, TaskState, TaskStatus, TaskStatusUpdateEvent, TextPart, ) -from splunk_ao.otel import SplunkAOSpanProcessor, add_splunk_ao_span_processor +from splunk_ao.otel import add_splunk_ao_span_processor from splunk_ao_a2a import A2AInstrumentor from langchain.agents import create_agent from langchain_core.tools import tool @@ -133,7 +152,7 @@ from typing_extensions import TypedDict # ---- Only 4 lines needed for full distributed tracing ---- provider = TracerProvider() -add_splunk_ao_span_processor(provider, SplunkAOSpanProcessor()) +add_splunk_ao_span_processor(provider) A2AInstrumentor().instrument(tracer_provider=provider, agent_name="orchestrator") LangchainInstrumentor().instrument(tracer_provider=provider) diff --git a/splunk-ao-a2a/examples/.env.example b/splunk-ao-a2a/examples/.env.example index c3eaf9fb..8baa8e19 100644 --- a/splunk-ao-a2a/examples/.env.example +++ b/splunk-ao-a2a/examples/.env.example @@ -1,5 +1,13 @@ +# Standalone Splunk AO SPLUNK_AO_API_KEY=your-splunk-ao-key SPLUNK_AO_CONSOLE_URL=https:// SPLUNK_AO_PROJECT=a2a-distributed-tracing-demo SPLUNK_AO_LOG_STREAM=dev + +# O11y cloud +# SPLUNK_AO_REALM=us1 +# SPLUNK_AO_SF_TOKEN=your-sf-ingest-token +# SPLUNK_AO_PROJECT=a2a-distributed-tracing-demo +# SPLUNK_AO_LOG_STREAM=dev + OPENAI_API_KEY=your-openai-key diff --git a/splunk-ao-a2a/tests/test_splunk_ao_compatibility.py b/splunk-ao-a2a/tests/test_splunk_ao_compatibility.py new file mode 100644 index 00000000..3913db80 --- /dev/null +++ b/splunk-ao-a2a/tests/test_splunk_ao_compatibility.py @@ -0,0 +1,80 @@ +from collections.abc import Sequence +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter, SpanExportResult + +from splunk_ao.deployment import DeploymentMode, StandaloneConfig +from splunk_ao.otel import SplunkAOOTLPExporter, SplunkAOSpanProcessor, add_splunk_ao_span_processor +from splunk_ao_a2a import _spans +from splunk_ao_a2a._constants import INSTRUMENTOR_NAME, INSTRUMENTOR_VERSION + + +class RecordingExporter(SpanExporter): + def __init__(self) -> None: + self.spans: list[ReadableSpan] = [] + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + self.spans.extend(spans) + return SpanExportResult.SUCCESS + + def shutdown(self) -> None: + pass + + +def test_a2a_native_span_uses_user_wired_deployment_aware_processor() -> None: + delegate = RecordingExporter() + captured_config: dict[str, object] = {} + + def exporter_factory(**kwargs: object) -> RecordingExporter: + captured_config.update(kwargs) + return delegate + + config = MagicMock() + config.resolve_deployment.return_value = DeploymentMode.STANDALONE + standalone = StandaloneConfig( + api_key="standalone-key", + console_url="https://console.example.com", + api_url="https://api.example.com", + ) + + with ( + patch("splunk_ao.otel.SplunkAOConfig.get", return_value=config), + patch("splunk_ao.otel.StandaloneConfig.from_env", return_value=standalone), + ): + exporter = SplunkAOOTLPExporter( + project="a2a-project", + logstream="a2a-log-stream", + _exporter_factory=exporter_factory, + ) + processor = SplunkAOSpanProcessor(SpanProcessor=SimpleSpanProcessor, _exporter=exporter) + provider = TracerProvider() + assert add_splunk_ao_span_processor(provider, processor) is processor + tracer = provider.get_tracer(INSTRUMENTOR_NAME, INSTRUMENTOR_VERSION) + try: + with tracer.start_as_current_span("a2a.client.send_message") as span: + _spans.set_client_attributes( + span, + SimpleNamespace(context_id="context-id", task_id="task-id"), + "SendMessage", + "orchestrator", + ) + finally: + provider.shutdown() + + assert captured_config == { + "endpoint": "https://api.example.com/otel/v1/traces", + "headers": { + "Splunk-AO-API-Key": "standalone-key", + "project": "a2a-project", + "logstream": "a2a-log-stream", + }, + } + exported = delegate.spans[0] + assert exported.instrumentation_scope.name == INSTRUMENTOR_NAME + assert exported.attributes["gen_ai.system"] == "a2a" + assert exported.attributes["a2a.rpc.method"] == "SendMessage" + assert exported.resource.attributes["splunk_ao.project.name"] == "a2a-project" + assert exported.resource.attributes["splunk_ao.logstream.name"] == "a2a-log-stream" + assert "splunk_ao.project.name" not in exported.attributes diff --git a/src/splunk_ao/otel.py b/src/splunk_ao/otel.py index b787136b..9e5ff2bc 100644 --- a/src/splunk_ao/otel.py +++ b/src/splunk_ao/otel.py @@ -2,20 +2,17 @@ import json import logging -import typing -from collections.abc import Generator +from collections.abc import Callable, Generator, Sequence from contextlib import contextmanager from contextvars import ContextVar from typing import Any, Protocol, cast -from urllib.parse import urljoin from opentelemetry import context, trace from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import Span, SpanProcessor -from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult from opentelemetry.trace import Tracer -from requests import Session from galileo_core.schemas.logging.span import RetrieverSpan, ToolSpan, WorkflowSpan from galileo_core.schemas.logging.span import Span as GalileoSpan @@ -29,7 +26,21 @@ _project_context, _session_id_context, ) -from splunk_ao.utils.env_helpers import _get_log_stream_or_default, _get_project_or_default +from splunk_ao.deployment import DeploymentMode, O11yConfig, StandaloneConfig +from splunk_ao.exporter import ( + RoutingAttrs, + resolve_o11y_exporter_config, + resolve_standalone_exporter_config, + routing_resource_attributes, +) +from splunk_ao.utils.env_helpers import ( + _get_log_stream_from_env, + _get_log_stream_id_from_env, + _get_log_stream_or_default, + _get_project_from_env, + _get_project_id_from_env, + _get_project_or_default, +) from splunk_ao.utils.retrievers import document_adapter logger = logging.getLogger(__name__) @@ -50,27 +61,130 @@ def get_tracer( _TRACE_PROVIDER_CONTEXT_VAR: ContextVar[TracerProvider | None] = ContextVar("galileo_trace_provider", default=None) -class SplunkAOOTLPExporter(OTLPSpanExporter): +ROUTING_ATTRIBUTE_KEYS = frozenset( + { + "splunk_ao.project.name", + "splunk_ao.project.id", + "splunk_ao.logstream.name", + "splunk_ao.logstream.id", + "splunk_ao.experiment.id", + } +) + + +def _resolve_name_or_id( + explicit_name: str | None, + explicit_id: str | None, + context_name: str | None, + environment_name: str | None, + environment_id: str | None, + default_name: str | None, +) -> tuple[str | None, str | None]: + """Resolve one immutable routing identity while preserving its supplied form.""" + if explicit_name: + return explicit_name, None + if explicit_id: + return None, explicit_id + if context_name: + return context_name, None + if environment_name: + return environment_name, None + if environment_id: + return None, environment_id + return default_name, None + + +def _resolve_routing( + deployment: DeploymentMode, + project: str | None, + project_id: str | None, + logstream: str | None, + log_stream_id: str | None, + experiment_id: str | None, +) -> RoutingAttrs: + """Capture routing once for one exporter without resolving names to IDs.""" + standalone = deployment == DeploymentMode.STANDALONE + project_name, resolved_project_id = _resolve_name_or_id( + project, + project_id, + _project_context.get(None), + _get_project_from_env(), + _get_project_id_from_env(), + _get_project_or_default(None) if standalone else None, + ) + log_stream_name, resolved_log_stream_id = _resolve_name_or_id( + logstream, + log_stream_id, + _log_stream_context.get(None), + _get_log_stream_from_env(), + _get_log_stream_id_from_env(), + _get_log_stream_or_default(None) if standalone else None, + ) + return RoutingAttrs( + project_name=project_name, + project_id=resolved_project_id, + log_stream_name=log_stream_name, + log_stream_id=resolved_log_stream_id, + experiment_id=experiment_id or _experiment_id_context.get(None), + ) + + +def _with_routing_resource(span: ReadableSpan, routing_resource: Resource) -> ReadableSpan: + """Return an immutable copy with authoritative routing in its Resource.""" + attributes = {key: value for key, value in (span.attributes or {}).items() if key not in ROUTING_ATTRIBUTE_KEYS} + source_resource = span.resource or Resource({}) + base_resource = Resource( + {key: value for key, value in source_resource.attributes.items() if key not in ROUTING_ATTRIBUTE_KEYS}, + schema_url=source_resource.schema_url, + ) + return ReadableSpan( + name=span.name, + context=span.context, + parent=span.parent, + resource=base_resource.merge(routing_resource), + attributes=attributes, + events=span.events, + links=span.links, + kind=span.kind, + instrumentation_info=span.instrumentation_info, + status=span.status, + start_time=span.start_time, + end_time=span.end_time, + instrumentation_scope=span.instrumentation_scope, + ) + + +class SplunkAOOTLPExporter(SpanExporter): """ - OpenTelemetry OTLP span exporter preconfigured for Galileo platform integration. + OpenTelemetry OTLP span exporter preconfigured for Splunk AO. - This exporter extends the standard OTLPSpanExporter with Galileo-specific - configuration and authentication. For most applications, consider using + This exporter wraps the standard OTLPSpanExporter with deployment-aware + configuration, authentication, and immutable routing. For most applications, use SplunkAOSpanProcessor instead, which provides a complete tracing solution. """ - _session: Session - - def __init__(self, project: str | None = None, logstream: str | None = None, **kwargs: Any) -> None: + def __init__( + self, + project: str | None = None, + project_id: str | None = None, + logstream: str | None = None, + log_stream_id: str | None = None, + experiment_id: str | None = None, + *, + _exporter_factory: Callable[..., SpanExporter] = OTLPSpanExporter, + **kwargs: Any, + ) -> None: """ - Initialize the Galileo OTLP exporter with authentication and endpoint configuration. + Initialize the Splunk AO OTLP exporter with deployment-aware configuration. Parameters ---------- - project : str, optional - Target Galileo project name. Falls back to SPLUNK_AO_PROJECT environment variable. - logstream : str, optional - Target logstream for trace organization. Uses default logstream if not specified. + project, project_id : str, optional + Target project name or ID. + logstream, log_stream_id : str, optional + Target log-stream name or ID. + experiment_id : str, optional + Target experiment ID. Takes precedence over log-stream routing. **kwargs Additional configuration options passed to the underlying OTLPSpanExporter. @@ -79,87 +193,33 @@ def __init__(self, project: str | None = None, logstream: str | None = None, **k ValueError When configuration is not properly initialized with required credentials. """ - # Get configuration from SplunkAOConfig config = SplunkAOConfig.get() + deployment = config.resolve_deployment() + self._routing = _resolve_routing(deployment, project, project_id, logstream, log_stream_id, experiment_id) + if deployment == DeploymentMode.O11Y: + exporter_config = resolve_o11y_exporter_config(O11yConfig.from_env(), self._routing) + else: + exporter_config = resolve_standalone_exporter_config(StandaloneConfig.from_env(), self._routing) + + self.project = self._routing.project_name + self.project_id = self._routing.project_id + self.logstream = self._routing.log_stream_name + self.log_stream_id = self._routing.log_stream_id + self.experiment_id = self._routing.experiment_id + self._routing_resource = Resource(routing_resource_attributes(self._routing)) + self._delegate = _exporter_factory(endpoint=exporter_config.endpoint, headers=exporter_config.headers, **kwargs) + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + """Export immutable copies with authoritative routing Resources.""" + return self._delegate.export(tuple(_with_routing_resource(span, self._routing_resource) for span in spans)) + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Flush the delegate exporter.""" + return self._delegate.force_flush(timeout_millis) - if not config.api_url: - # This should never happen, but we'll raise an error just in case - raise ValueError("API URL is required.") - - # Get API URL and construct OTLP endpoint - base_url = str(config.api_url) - # Ensure base_url ends with / for proper joining - if not base_url.endswith("/"): - base_url += "/" - endpoint: str = urljoin(base_url, "otel/v1/traces") - api_key = config.api_key.get_secret_value() if config.api_key else None - - if not api_key: - raise ValueError("API key is required.") - - # Resolve project and logstream: param first, then context var, then env var with default fallback - ctx_project = project if project is not None else _project_context.get(None) - ctx_logstream = logstream if logstream is not None else _log_stream_context.get(None) - - self.project = _get_project_or_default(ctx_project) - self.logstream = _get_log_stream_or_default(ctx_logstream) - - exporter_headers = {"Splunk-AO-API-Key": api_key, "project": self.project, "logstream": self.logstream} - - super().__init__(endpoint=endpoint, headers=exporter_headers, **kwargs) - - def export(self, spans: typing.Sequence[Any]) -> Any: - """Override export to set resource attributes from span attributes before serialization.""" - is_experiment = False - for span in spans: - # Read from span attributes (set during on_start when context was available) - project = span.attributes.get("splunk_ao.project.name") - logstream = span.attributes.get("splunk_ao.logstream.name") - session_id = span.attributes.get("splunk_ao.session.id") - experiment_id = span.attributes.get("splunk_ao.experiment.id") - dataset_input = span.attributes.get("splunk_ao.dataset.input") - dataset_output = span.attributes.get("splunk_ao.dataset.output") - dataset_metadata = span.attributes.get("splunk_ao.dataset.metadata") - - # Build resource attributes dict, filtering out None values - resource_attrs = {} - if project: - resource_attrs["splunk_ao.project.name"] = project - # We can only have either logstream or experiment, if it's an experiment we want to prioritize it. - if logstream and not experiment_id: - resource_attrs["splunk_ao.logstream.name"] = logstream - if session_id: - resource_attrs["splunk_ao.session.id"] = session_id - if experiment_id: - resource_attrs["splunk_ao.experiment.id"] = experiment_id - is_experiment = True - if dataset_input: - resource_attrs["splunk_ao.dataset.input"] = dataset_input - if dataset_output: - resource_attrs["splunk_ao.dataset.output"] = dataset_output - if dataset_metadata: - resource_attrs["splunk_ao.dataset.metadata"] = dataset_metadata - - if resource_attrs: - # Merge new attributes into span's resource - new_resource = span.resource.merge(Resource(resource_attrs)) - # Mutate the internal _resource (ReadableSpan stores it there) - span._resource = new_resource - - # for the last span update the headers - if spans: - last_span = spans[-1] - self._session.headers.update( - { - "project": last_span.attributes.get("splunk_ao.project.name"), - "logstream": last_span.attributes.get("splunk_ao.logstream.name"), - } - ) - if is_experiment: - self._session.headers.update({"experimentid": last_span.attributes.get("splunk_ao.experiment.id")}) - self._session.headers.pop("logstream", None) # Remove logstream header if experiment is present - - return super().export(spans) + def shutdown(self) -> None: + """Shut down the delegate exporter.""" + self._delegate.shutdown() class SplunkAOSpanProcessor(SpanProcessor): @@ -174,35 +234,52 @@ class SplunkAOSpanProcessor(SpanProcessor): -------- >>> from opentelemetry.sdk.trace import TracerProvider >>> tracer_provider = TracerProvider() - >>> processor = SplunkAOSpanProcessor(project="my-project") - >>> add_splunk_ao_span_processor(tracer_provider, processor) + >>> processor = add_splunk_ao_span_processor(tracer_provider, project="my-project") """ def __init__( - self, project: str | None = None, logstream: str | None = None, SpanProcessor: type | None = None, **kwargs: Any + self, + project: str | None = None, + project_id: str | None = None, + logstream: str | None = None, + log_stream_id: str | None = None, + experiment_id: str | None = None, + SpanProcessor: type | None = None, + *, + _exporter: SpanExporter | None = None, + _exporter_factory: Callable[..., SpanExporter] = OTLPSpanExporter, + **kwargs: Any, ) -> None: """ Initialize the Galileo span processor with export configuration. Parameters ---------- - project : str, optional - Target Galileo project for trace data. Falls back to SPLUNK_AO_PROJECT environment variable. - logstream : str, optional - Target logstream for trace organization. Uses default logstream if not specified. + project, project_id : str, optional + Target project name or ID. + logstream, log_stream_id : str, optional + Target log-stream name or ID. + experiment_id : str, optional + Target experiment ID. Takes precedence over log-stream routing. SpanProcessor : type, optional Custom span processor class. Defaults to BatchSpanProcessor for optimal performance. """ - # Resolve project and logstream: param first, then context var, then env var with default fallback - ctx_project = project if project is not None else _project_context.get(None) - ctx_logstream = logstream if logstream is not None else _log_stream_context.get(None) - - self._project = _get_project_or_default(ctx_project) - self._logstream = _get_log_stream_or_default(ctx_logstream) - - # Create the exporter using the config-based approach - self._exporter = SplunkAOOTLPExporter(**kwargs) + self._exporter = ( + _exporter + if _exporter is not None + else SplunkAOOTLPExporter( + project=project, + project_id=project_id, + logstream=logstream, + log_stream_id=log_stream_id, + experiment_id=experiment_id, + _exporter_factory=_exporter_factory, + **kwargs, + ) + ) + self._project = getattr(self._exporter, "project", project) + self._logstream = getattr(self._exporter, "logstream", logstream) if SpanProcessor is None: SpanProcessor = BatchSpanProcessor @@ -211,20 +288,8 @@ def __init__( def on_start(self, span: Span, parent_context: context.Context | None = None) -> None: """Handle span start events by delegating to the underlying processor.""" - # Set Galileo context attributes on the span - # Use context var if set and not None, otherwise fall back to instance defaults - project = _project_context.get(None) or self._project - log_stream = _log_stream_context.get(None) or self._logstream - experiment_id = _experiment_id_context.get(None) session_id = _session_id_context.get(None) - if project: - span.set_attribute("splunk_ao.project.name", project) - # We can only have either logstream or experiment, if it's an experiment we want to prioritize it. - if log_stream and not experiment_id: - span.set_attribute("splunk_ao.logstream.name", log_stream) - if experiment_id: - span.set_attribute("splunk_ao.experiment.id", experiment_id) if session_id: span.set_attribute("splunk_ao.session.id", session_id) @@ -238,26 +303,22 @@ def on_start(self, span: Span, parent_context: context.Context | None = None) -> self._processor.on_start(span, parent_context) - def on_end(self, span: Span) -> None: + def on_end(self, span: ReadableSpan) -> None: """Handle span completion events by delegating to the underlying processor.""" self._processor.on_end(span) def shutdown(self) -> None: """Gracefully shutdown the processor and flush any remaining spans.""" self._processor.shutdown() - logger.info( - "Galileo span processor shutdown for project %s and logstream %s", - self.exporter.project, - self.exporter.logstream, - ) + logger.info("Splunk AO span processor shutdown for project %s and logstream %s", self._project, self._logstream) - def force_flush(self, timeout_millis: int = 40000) -> None: + def force_flush(self, timeout_millis: int = 40000) -> bool: """Force immediate export of all pending spans with specified timeout.""" return self._processor.force_flush(timeout_millis) @property - def exporter(self) -> SplunkAOOTLPExporter: - """Access to the underlying Galileo OTLP exporter instance.""" + def exporter(self) -> SpanExporter: + """Access to the underlying Splunk AO OTLP exporter instance.""" return self._exporter @property @@ -266,10 +327,17 @@ def processor(self) -> SpanProcessor: return self._processor -def add_splunk_ao_span_processor(tracer_provider: TracerProvider, processor: SplunkAOSpanProcessor) -> None: - """Add the Galileo span processor to the tracer provider.""" - tracer_provider.add_span_processor(processor) +def add_splunk_ao_span_processor( + tracer_provider: TracerProvider, processor: SplunkAOSpanProcessor | None = None, **processor_kwargs: Any +) -> SplunkAOSpanProcessor: + """Construct or accept, register, and return a Splunk AO span processor.""" + if processor is not None and processor_kwargs: + raise ValueError("processor_kwargs cannot be used with an existing processor") + + resolved_processor = processor if processor is not None else SplunkAOSpanProcessor(**processor_kwargs) + tracer_provider.add_span_processor(resolved_processor) _TRACE_PROVIDER_CONTEXT_VAR.set(tracer_provider) + return resolved_processor def _set_retriever_span_attributes(span: trace.Span, galileo_span: RetrieverSpan) -> None: @@ -363,7 +431,6 @@ def start_splunk_ao_span(galileo_span: GalileoSpan) -> Generator[trace.Span, Any tracer = tracer_provider.get_tracer("galileo-tracer") with tracer.start_as_current_span(galileo_span.name) as span: yield span - span.set_attribute("gen_ai.system", "galileo-otel") # Set dataset attributes for ground truth/reference output support _apply_dataset_attributes( span, galileo_span.dataset_input, galileo_span.dataset_output, galileo_span.dataset_metadata diff --git a/tests/test_otel.py b/tests/test_otel.py index 372269d9..5b114048 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -1,9 +1,7 @@ import json -import os from unittest.mock import Mock, patch import pytest -from pydantic import SecretStr from galileo_core.schemas.logging.llm import Message, MessageRole from galileo_core.schemas.logging.span import ToolSpan, WorkflowSpan @@ -18,6 +16,7 @@ _session_id_context, splunk_ao_dataset_context, ) +from splunk_ao.deployment import DeploymentMode, StandaloneConfig from splunk_ao.otel import ( _TRACE_PROVIDER_CONTEXT_VAR, SplunkAOOTLPExporter, @@ -28,93 +27,6 @@ ) -class TestSplunkAOOTLPExporter: - """Test suite for SplunkAOOTLPExporter class.""" - - @pytest.fixture - def clear_env_vars(self): - """Clear relevant environment variables and context vars for clean test state.""" - env_vars = ["SPLUNK_AO_API_KEY", "SPLUNK_AO_CONSOLE_URL", "SPLUNK_AO_PROJECT", "SPLUNK_AO_LOG_STREAM"] - original_values = {var: os.environ.pop(var, None) for var in env_vars} - _project_context.set(None) - _log_stream_context.set(None) - - yield - - for var, value in original_values.items(): - if value is not None: - os.environ[var] = value - _project_context.set(None) - _log_stream_context.set(None) - - @pytest.fixture - def mock_config(self): - """Create a mock config with default values.""" - with patch("splunk_ao.otel.SplunkAOConfig.get") as mock_config_get: - config = Mock() - config.api_url = "https://api.galileo.ai" - config.api_key = SecretStr("test-key") - mock_config_get.return_value = config - yield config - - @patch("splunk_ao.otel.OTLPSpanExporter.__init__", return_value=None) - def test_init_and_parameter_priority(self, mock_otlp_init, mock_config, clear_env_vars): - """Test initialization with params, env vars, and their priority.""" - # Test with explicit params - exporter = SplunkAOOTLPExporter(project="param-project", logstream="param-logstream", timeout=30) - assert exporter.project == "param-project" - assert exporter.logstream == "param-logstream" - - call_kwargs = mock_otlp_init.call_args[1] - assert call_kwargs["headers"]["project"] == "param-project" - assert call_kwargs["headers"]["logstream"] == "param-logstream" - assert call_kwargs["timeout"] == 30 - - # Test that params override env vars - mock_otlp_init.reset_mock() - with patch.dict(os.environ, {"SPLUNK_AO_PROJECT": "env-project", "SPLUNK_AO_LOG_STREAM": "env-logstream"}): - exporter = SplunkAOOTLPExporter(project="param-project", logstream="param-logstream") - assert exporter.project == "param-project" # Param wins over env - - @patch("splunk_ao.otel.OTLPSpanExporter.__init__", return_value=None) - def test_init_with_env_variables(self, mock_otlp_init, mock_config, clear_env_vars): - """Test initialization using environment variables.""" - with patch.dict(os.environ, {"SPLUNK_AO_PROJECT": "env-project", "SPLUNK_AO_LOG_STREAM": "env-logstream"}): - exporter = SplunkAOOTLPExporter() - assert exporter.project == "env-project" - assert exporter.logstream == "env-logstream" - - @patch("splunk_ao.otel.OTLPSpanExporter.__init__", return_value=None) - def test_init_uses_default_project(self, mock_otlp_init, mock_config, clear_env_vars): - """Test default project name is used when no project is provided.""" - SplunkAOOTLPExporter() - - call_kwargs = mock_otlp_init.call_args[1] - assert call_kwargs["headers"]["project"] == "default" - assert call_kwargs["headers"]["logstream"] == "default" - - @patch("splunk_ao.otel.OTLPSpanExporter.__init__", return_value=None) - @pytest.mark.parametrize( - "api_url,expected_endpoint", - [ - ("https://api.galileo.ai", "https://api.galileo.ai/otel/v1/traces"), - ("https://api.galileo.ai/", "https://api.galileo.ai/otel/v1/traces"), - ("http://localhost:8080", "http://localhost:8080/otel/v1/traces"), - ], - ) - def test_url_construction(self, mock_otlp_init, api_url, expected_endpoint, mock_config, clear_env_vars): - """Test URL construction with various formats.""" - mock_config.api_url = api_url - SplunkAOOTLPExporter() - assert mock_otlp_init.call_args[1]["endpoint"] == expected_endpoint - - def test_init_missing_api_key_raises_error(self, mock_config, clear_env_vars): - """Test that missing API key raises ValueError.""" - mock_config.api_key = None - with pytest.raises(ValueError, match="API key is required"): - SplunkAOOTLPExporter() - - class TestSplunkAOSpanProcessor: """Test suite for SplunkAOSpanProcessor class.""" @@ -234,24 +146,35 @@ def test_init_passes_all_parameters_to_exporter(self, mock_processor_setup): class TestOTelIntegration: """Integration tests for OpenTelemetry functionality.""" - @patch("splunk_ao.otel.BatchSpanProcessor") - @patch("splunk_ao.otel.OTLPSpanExporter.__init__", return_value=None) - @patch("splunk_ao.otel.SplunkAOConfig.get") - def test_exporter_and_processor_integration(self, mock_config_get, mock_otlp_init, mock_batch_processor): + def test_exporter_and_processor_integration(self): """Test that SplunkAOSpanProcessor correctly integrates with SplunkAOOTLPExporter.""" mock_batch_instance = Mock() - mock_batch_processor.return_value = mock_batch_instance - - # Mock the config mock_config = Mock() - mock_config.api_url = "https://api.galileo.ai" - mock_config.api_key = SecretStr("test-key") - mock_config_get.return_value = mock_config - - processor = SplunkAOSpanProcessor(project="integration-test", logstream="integration-logstream") + mock_config.resolve_deployment.return_value = DeploymentMode.STANDALONE + standalone = StandaloneConfig( + api_key="test-key", console_url="https://console.example.com", api_url="https://api.example.com" + ) + experiment_token = _experiment_id_context.set(None) + try: + with ( + patch("splunk_ao.otel.BatchSpanProcessor", return_value=mock_batch_instance) as mock_batch_processor, + patch("splunk_ao.otel.OTLPSpanExporter.__init__", return_value=None) as mock_otlp_init, + patch("splunk_ao.otel.SplunkAOConfig.get", return_value=mock_config), + patch("splunk_ao.otel.StandaloneConfig.from_env", return_value=standalone), + ): + processor = SplunkAOSpanProcessor(project="integration-test", logstream="integration-logstream") + finally: + _experiment_id_context.reset(experiment_token) # Verify the exporter was created and passed to the processor - assert mock_otlp_init.called + mock_otlp_init.assert_called_once_with( + endpoint="https://api.example.com/otel/v1/traces", + headers={ + "Splunk-AO-API-Key": "test-key", + "project": "integration-test", + "logstream": "integration-logstream", + }, + ) mock_batch_processor.assert_called_once() # Verify the processor has access to both components @@ -283,68 +206,43 @@ def mock_processor_deps(self): mock_batch.return_value = Mock() yield {"exporter": mock_exp, "batch": mock_batch} - @pytest.fixture - def mock_exporter(self): - """Create a mock exporter for export tests.""" - with ( - patch("splunk_ao.otel.OTLPSpanExporter.__init__", return_value=None), - patch("splunk_ao.otel.SplunkAOConfig.get") as mock_config_get, - ): - config = Mock() - config.api_url = "https://api.galileo.ai" - config.api_key = SecretStr("test-key") - mock_config_get.return_value = config - yield SplunkAOOTLPExporter(project="test-project", logstream="test-logstream") - - @patch("splunk_ao.otel.OTLPSpanExporter.__init__", return_value=None) - @patch("splunk_ao.otel.SplunkAOConfig.get") - def test_exporter_context_vars_and_override(self, mock_config_get, mock_otlp_init, reset_decorator_context): + def test_exporter_context_vars_and_override(self, reset_decorator_context): """Test exporter reads from context vars and params override them.""" mock_config = Mock() - mock_config.api_url = "https://api.galileo.ai" - mock_config.api_key = SecretStr("test-key") - mock_config_get.return_value = mock_config + mock_config.resolve_deployment.return_value = DeploymentMode.STANDALONE + standalone = StandaloneConfig( + api_key="test-key", console_url="https://console.example.com", api_url="https://api.example.com" + ) # Set context variables _project_context.set("context-project") _log_stream_context.set("context-logstream") - # Exporter uses context values - exporter = SplunkAOOTLPExporter() - assert exporter.project == "context-project" - assert exporter.logstream == "context-logstream" + with ( + patch("splunk_ao.otel.OTLPSpanExporter.__init__", return_value=None), + patch("splunk_ao.otel.SplunkAOConfig.get", return_value=mock_config), + patch("splunk_ao.otel.StandaloneConfig.from_env", return_value=standalone), + ): + # Exporter uses context values + exporter = SplunkAOOTLPExporter() + assert exporter.project == "context-project" + assert exporter.logstream == "context-logstream" - # Explicit params override context - mock_otlp_init.reset_mock() - exporter2 = SplunkAOOTLPExporter(project="param-project", logstream="param-logstream") - assert exporter2.project == "param-project" - assert exporter2.logstream == "param-logstream" + # Explicit params override context + exporter2 = SplunkAOOTLPExporter(project="param-project", logstream="param-logstream") + assert exporter2.project == "param-project" + assert exporter2.logstream == "param-logstream" - def test_processor_context_and_fallback(self, mock_processor_deps, reset_decorator_context): - """Test processor reads context vars and uses init values as fallback.""" - # Test context vars + def test_processor_captures_context_at_exporter_construction(self, mock_processor_deps, reset_decorator_context): + """Test processor passes routing inputs to its immutable exporter.""" _project_context.set("context-project") _log_stream_context.set("context-logstream") - processor = SplunkAOSpanProcessor() - assert processor._project == "context-project" - assert processor._logstream == "context-logstream" - - # Test fallback to init values when context is empty - _project_context.set(None) - _log_stream_context.set(None) + SplunkAOSpanProcessor() + mock_processor_deps["exporter"].assert_called_once() - processor2 = SplunkAOSpanProcessor(project="init-project", logstream="init-logstream") - assert processor2._project == "init-project" - assert processor2._logstream == "init-logstream" - - def test_processor_on_start_sets_span_attributes(self, mock_processor_deps, reset_decorator_context, monkeypatch): - """Test on_start sets context attributes on spans, handling None values.""" - # Pin env vars for this test to avoid flakiness from parallel workers - monkeypatch.setenv("SPLUNK_AO_PROJECT", "test-project") - monkeypatch.setenv("SPLUNK_AO_LOG_STREAM", "test-log-stream") - - # Given: all context vars set (experiment_id takes priority over logstream) + def test_processor_on_start_sets_content_not_routing_attributes(self, mock_processor_deps, reset_decorator_context): + """Test on_start keeps session content and omits request routing.""" _project_context.set("test-project") _log_stream_context.set("test-logstream") _experiment_id_context.set("test-experiment") @@ -356,101 +254,11 @@ def test_processor_on_start_sets_span_attributes(self, mock_processor_deps, rese # When: the processor starts a span processor.on_start(mock_span, None) - # Then: experiment_id is set and logstream is excluded (experiment takes priority) - assert mock_span.set_attribute.call_count == 3 + assert mock_span.set_attribute.call_count == 1 actual_calls = {(args[0], args[1]) for args, _ in mock_span.set_attribute.call_args_list} - assert ("splunk_ao.project.name", "test-project") in actual_calls assert ("splunk_ao.session.id", "test-session") in actual_calls - assert ("splunk_ao.experiment.id", "test-experiment") in actual_calls - assert ("splunk_ao.logstream.name", "test-logstream") not in actual_calls - - # Given: context vars are None, falling back to env vars - _log_stream_context.set(None) - _experiment_id_context.set(None) - _session_id_context.set(None) - - monkeypatch = pytest.MonkeyPatch() - monkeypatch.setenv("SPLUNK_AO_LOG_STREAM", "test-log-stream") - try: - processor2 = SplunkAOSpanProcessor() - mock_span2 = Mock() - processor2.on_start(mock_span2, None) - - # Then: project and logstream are set from env var fallbacks - assert mock_span2.set_attribute.call_count == 2 - actual_calls = {(args[0], args[1]) for args, _ in mock_span2.set_attribute.call_args_list} - assert ("splunk_ao.project.name", "test-project") in actual_calls - # Falls back to SPLUNK_AO_LOG_STREAM env var - assert ("splunk_ao.logstream.name", "test-log-stream") in actual_calls - finally: - monkeypatch.undo() - - @patch("splunk_ao.otel.OTLPSpanExporter.export") - @patch("splunk_ao.otel.Resource") - def test_exporter_export_merges_resource_attributes(self, mock_resource_class, mock_parent_export): - """Test export merges Galileo attributes into resource, handles partial/no attributes.""" - # Create a real exporter with mocked dependencies - with ( - patch("splunk_ao.otel.OTLPSpanExporter.__init__", return_value=None), - patch("splunk_ao.otel.SplunkAOConfig.get") as mock_config_get, - ): - config = Mock() - config.api_url = "https://api.galileo.ai" - config.api_key = SecretStr("test-key") - mock_config_get.return_value = config - - exporter = SplunkAOOTLPExporter(project="test-project", logstream="test-logstream") - # Mock the _session attribute that export() uses - exporter._session = Mock() - exporter._session.headers = {} - - # Given: a span with experiment_id present (logstream should be excluded) - mock_span = Mock() - mock_span.attributes = { - "splunk_ao.project.name": "span-project", - "splunk_ao.logstream.name": "span-logstream", - "splunk_ao.session.id": "span-session", - "splunk_ao.experiment.id": "span-experiment", - "other.attribute": "value", - } - mock_merged_resource = Mock() - mock_span.resource = Mock() - mock_span.resource.merge.return_value = mock_merged_resource - mock_new_resource = Mock() - mock_resource_class.return_value = mock_new_resource - - exporter.export([mock_span]) - - # Then: resource was created with Galileo attributes (logstream excluded when experiment present) - call_args = mock_resource_class.call_args[0][0] - assert "splunk_ao.project.name" in call_args - assert call_args["splunk_ao.project.name"] == "span-project" - # Then: logstream is not included because experiment takes priority - assert "splunk_ao.logstream.name" not in call_args - assert "splunk_ao.session.id" in call_args - assert "splunk_ao.experiment.id" in call_args - assert "other.attribute" not in call_args - - # Verify resource was merged and span._resource was updated - mock_span.resource.merge.assert_called_once_with(mock_new_resource) - assert mock_span._resource == mock_merged_resource - - # Verify parent export was called - mock_parent_export.assert_called_once_with([mock_span]) - - # Test with no Galileo attributes - mock_parent_export.reset_mock() - mock_resource_class.reset_mock() - mock_span2 = Mock() - mock_span2.attributes = {"other.attribute": "value"} - mock_span2.resource = Mock() - - exporter.export([mock_span2]) - - # No resource should be created when there are no Galileo attributes - mock_resource_class.assert_not_called() - mock_span2.resource.merge.assert_not_called() - mock_parent_export.assert_called_once_with([mock_span2]) + routing_keys = {"splunk_ao.project.name", "splunk_ao.logstream.name", "splunk_ao.experiment.id"} + assert not routing_keys.intersection(key for key, _ in actual_calls) class TestSetToolSpanAttributes: @@ -557,9 +365,9 @@ def test_start_splunk_ao_span_dispatches_tool_span(self): with start_splunk_ao_span(tool_span) as span: assert span is mock_otel_span - # Then: the span has gen_ai.system set and tool-specific attributes + # Then: tool-specific attributes are retained without an SDK provider injection calls = {args[0]: args[1] for args, _ in mock_otel_span.set_attribute.call_args_list} - assert calls["gen_ai.system"] == "galileo-otel" + assert "gen_ai.system" not in calls assert calls["gen_ai.operation.name"] == "execute_tool" assert calls["gen_ai.tool.name"] == "my-tool" assert calls["gen_ai.tool.call.arguments"] == "tool input data" @@ -584,9 +392,9 @@ def test_start_splunk_ao_span_tool_span_with_none_output(self): with start_splunk_ao_span(tool_span) as span: assert span is mock_otel_span - # Then: gen_ai.system, operation name, tool name, tool arguments, and input are set (no output or tool_call_id) + # Then: tool attributes are set without output, call ID, or an SDK provider injection calls = {args[0]: args[1] for args, _ in mock_otel_span.set_attribute.call_args_list} - assert calls["gen_ai.system"] == "galileo-otel" + assert "gen_ai.system" not in calls assert calls["gen_ai.operation.name"] == "execute_tool" assert calls["gen_ai.tool.name"] == "minimal-tool" assert calls["gen_ai.tool.call.arguments"] == "just input" @@ -707,10 +515,10 @@ def test_workflow_span_in_start_splunk_ao_span(self, mock_dependencies): with start_splunk_ao_span(workflow_span): pass - # Then: WorkflowSpan attributes should be set - assert mock_span.set_attribute.call_count >= 2 # gen_ai.system + at least input + # Then: WorkflowSpan attributes should be set without an SDK provider injection + assert mock_span.set_attribute.call_count >= 2 system_call = [call for call in mock_span.set_attribute.call_args_list if call[0][0] == "gen_ai.system"] - assert len(system_call) == 1 + assert system_call == [] class TestDatasetContext: @@ -811,54 +619,6 @@ def test_processor_on_start_sets_dataset_attributes(self, mock_processor_deps, r assert ("splunk_ao.dataset.output", "expected answer") in actual_calls assert ("splunk_ao.dataset.metadata", json.dumps({"source": "test_dataset"})) in actual_calls - @patch("splunk_ao.otel.OTLPSpanExporter.export") - @patch("splunk_ao.otel.Resource") - def test_exporter_export_merges_dataset_attributes( - self, mock_resource_class, mock_parent_export, reset_dataset_context - ): - """Test that export merges dataset attributes from span into resource.""" - # Given: an exporter with mocked dependencies - with ( - patch("splunk_ao.otel.OTLPSpanExporter.__init__", return_value=None), - patch("splunk_ao.otel.SplunkAOConfig.get") as mock_config_get, - ): - config = Mock() - config.api_url = "https://api.galileo.ai" - config.api_key = SecretStr("test-key") - mock_config_get.return_value = config - - exporter = SplunkAOOTLPExporter(project="test-project", logstream="test-logstream") - exporter._session = Mock() - exporter._session.headers = {} - - # Given: a span with dataset attributes (set during on_start) - mock_span = Mock() - mock_span.attributes = { - "splunk_ao.project.name": "test-project", - "splunk_ao.logstream.name": "test-logstream", - "splunk_ao.dataset.input": "test input", - "splunk_ao.dataset.output": "expected output", - "splunk_ao.dataset.metadata": json.dumps({"key": "value"}), - } - mock_merged_resource = Mock() - mock_span.resource = Mock() - mock_span.resource.merge.return_value = mock_merged_resource - mock_new_resource = Mock() - mock_resource_class.return_value = mock_new_resource - - # When: export is called - exporter.export([mock_span]) - - # Then: Resource is created with dataset attributes - resource_call_kwargs = mock_resource_class.call_args[0][0] - assert resource_call_kwargs["splunk_ao.dataset.input"] == "test input" - assert resource_call_kwargs["splunk_ao.dataset.output"] == "expected output" - assert resource_call_kwargs["splunk_ao.dataset.metadata"] == json.dumps({"key": "value"}) - - # Then: resource was merged into the span - mock_span.resource.merge.assert_called_once_with(mock_new_resource) - assert mock_span._resource == mock_merged_resource - def test_splunk_ao_dataset_context_partial_values(self, reset_dataset_context): """Test that splunk_ao_dataset_context works with partial values.""" # When: only some values are provided diff --git a/tests/test_otel_native_paths.py b/tests/test_otel_native_paths.py new file mode 100644 index 00000000..34daa22b --- /dev/null +++ b/tests/test_otel_native_paths.py @@ -0,0 +1,387 @@ +from collections.abc import Sequence +from unittest.mock import MagicMock, patch + +import pytest +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import Event, ReadableSpan +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from opentelemetry.trace import Link, SpanContext, SpanKind, TraceFlags +from opentelemetry.trace.status import Status, StatusCode + +from splunk_ao.decorator import ( + _dataset_input_context, + _dataset_metadata_context, + _dataset_output_context, + _experiment_id_context, + _log_stream_context, + _project_context, + _session_id_context, +) +from splunk_ao.deployment import DeploymentMode, O11yConfig, StandaloneConfig +from splunk_ao.otel import ( + _TRACE_PROVIDER_CONTEXT_VAR, + SplunkAOOTLPExporter, + SplunkAOSpanProcessor, + add_splunk_ao_span_processor, +) +from splunk_ao.shared.exceptions import MissingConfigurationError + +ROUTING_KEYS = { + "splunk_ao.project.name", + "splunk_ao.project.id", + "splunk_ao.logstream.name", + "splunk_ao.logstream.id", + "splunk_ao.experiment.id", +} + + +class RecordingExporter(SpanExporter): + def __init__(self) -> None: + self.spans: list[ReadableSpan] = [] + self.force_flush_timeouts: list[int] = [] + self.shutdown_calls = 0 + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + self.spans.extend(spans) + return SpanExportResult.SUCCESS + + def force_flush(self, timeout_millis: int = 30000) -> bool: + self.force_flush_timeouts.append(timeout_millis) + return True + + def shutdown(self) -> None: + self.shutdown_calls += 1 + + +class RecordingExporterFactory: + def __init__(self) -> None: + self.calls: list[dict[str, object]] = [] + self.exporter = RecordingExporter() + + def __call__(self, **kwargs: object) -> RecordingExporter: + self.calls.append(kwargs) + return self.exporter + + +class RecordingSpanProcessor: + def __init__(self, exporter: SpanExporter) -> None: + self.exporter = exporter + self.started: list[object] = [] + self.ended: list[object] = [] + self.shutdown_calls = 0 + + def on_start(self, span: object, parent_context: object | None = None) -> None: + self.started.append((span, parent_context)) + + def on_end(self, span: object) -> None: + self.ended.append(span) + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return True + + def shutdown(self) -> None: + self.shutdown_calls += 1 + + +@pytest.fixture(autouse=True) +def reset_otel_context(monkeypatch: pytest.MonkeyPatch): + contexts = ( + _project_context, + _log_stream_context, + _experiment_id_context, + _session_id_context, + _dataset_input_context, + _dataset_output_context, + _dataset_metadata_context, + ) + for context_var in contexts: + context_var.set(None) + _TRACE_PROVIDER_CONTEXT_VAR.set(None) + for name in ("SPLUNK_AO_PROJECT", "SPLUNK_AO_PROJECT_ID", "SPLUNK_AO_LOG_STREAM", "SPLUNK_AO_LOG_STREAM_ID"): + monkeypatch.delenv(name, raising=False) + yield + for context_var in contexts: + context_var.set(None) + _TRACE_PROVIDER_CONTEXT_VAR.set(None) + + +def make_span() -> ReadableSpan: + context = SpanContext( + trace_id=0x1234567890ABCDEF1234567890ABCDEF, + span_id=0x1234567890ABCDEF, + is_remote=False, + trace_flags=TraceFlags.SAMPLED, + ) + parent = SpanContext( + trace_id=context.trace_id, span_id=0xFEDCBA0987654321, is_remote=True, trace_flags=TraceFlags.SAMPLED + ) + linked_context = SpanContext( + trace_id=0xABCDEF1234567890ABCDEF1234567890, + span_id=0xABCDEF1234567890, + is_remote=False, + trace_flags=TraceFlags.SAMPLED, + ) + return ReadableSpan( + name="upstream-span", + context=context, + parent=parent, + resource=Resource( + {"service.name": "checkout", "splunk_ao.project.name": "stale-resource-project"}, + schema_url="https://opentelemetry.io/schemas/1.38.0", + ), + attributes={ + "gen_ai.request.model": "gpt-4o", + "gen_ai.provider.name": "openai", + "gen_ai.system": "legacy-upstream-provider", + "custom.attribute": "preserved", + "splunk_ao.project.name": "stale-span-project", + }, + events=(Event("event", {"event.attribute": "value"}, timestamp=3),), + links=(Link(linked_context, {"link.attribute": "value"}),), + kind=SpanKind.CLIENT, + status=Status(StatusCode.OK), + start_time=1, + end_time=2, + instrumentation_scope=InstrumentationScope("upstream.instrumentor", "1.0.0"), + ) + + +def build_exporter(mode: DeploymentMode, factory: RecordingExporterFactory, **routing: str) -> SplunkAOOTLPExporter: + config = MagicMock() + config.resolve_deployment.return_value = mode + standalone = StandaloneConfig( + api_key="standalone-key", console_url="https://console.example.com", api_url="https://api.example.com" + ) + o11y = O11yConfig(realm="us1", sf_token="o11y-token") + with ( + patch("splunk_ao.otel.SplunkAOConfig.get", return_value=config), + patch("splunk_ao.otel.StandaloneConfig.from_env", return_value=standalone), + patch("splunk_ao.otel.O11yConfig.from_env", return_value=o11y), + ): + return SplunkAOOTLPExporter(_exporter_factory=factory, **routing) + + +def test_standalone_exporter_uses_shared_config_and_name_routing() -> None: + factory = RecordingExporterFactory() + exporter = build_exporter(DeploymentMode.STANDALONE, factory, project="payments", logstream="production") + + assert factory.calls == [ + { + "endpoint": "https://api.example.com/otel/v1/traces", + "headers": {"Splunk-AO-API-Key": "standalone-key", "project": "payments", "logstream": "production"}, + } + ] + exporter.shutdown() + + +def test_o11y_exporter_supports_id_and_experiment_routing() -> None: + factory = RecordingExporterFactory() + exporter = build_exporter( + DeploymentMode.O11Y, + factory, + project_id="project-id", + log_stream_id="ignored-log-stream-id", + experiment_id="experiment-id", + ) + + assert factory.calls[0] == { + "endpoint": "https://ingest.us1.observability.splunkcloud.com/v2/trace/otlp", + "headers": {"X-SF-Token": "o11y-token", "projectid": "project-id", "experimentid": "experiment-id"}, + } + exporter.shutdown() + + +def test_o11y_exporter_allows_auth_only_routing() -> None: + factory = RecordingExporterFactory() + exporter = build_exporter(DeploymentMode.O11Y, factory) + + assert factory.calls[0]["headers"] == {"X-SF-Token": "o11y-token"} + exporter.export((make_span(),)) + exported = factory.exporter.spans[0] + assert not ROUTING_KEYS.intersection(exported.resource.attributes) + assert not ROUTING_KEYS.intersection(exported.attributes or {}) + exporter.shutdown() + + +def test_o11y_exporter_rejects_crud_only_config_before_delegate_construction() -> None: + factory = RecordingExporterFactory() + config = MagicMock() + config.resolve_deployment.return_value = DeploymentMode.O11Y + crud_only = O11yConfig(realm="us1", sf_api_token="api-token") + + with ( + patch("splunk_ao.otel.SplunkAOConfig.get", return_value=config), + patch("splunk_ao.otel.O11yConfig.from_env", return_value=crud_only), + pytest.raises(MissingConfigurationError, match="SPLUNK_AO_SF_TOKEN"), + ): + SplunkAOOTLPExporter(_exporter_factory=factory) + + assert factory.calls == [] + + +def test_explicit_id_routing_precedes_context_and_environment_names(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SPLUNK_AO_PROJECT", "environment-project") + monkeypatch.setenv("SPLUNK_AO_LOG_STREAM", "environment-log-stream") + _project_context.set("context-project") + _log_stream_context.set("context-log-stream") + factory = RecordingExporterFactory() + + exporter = build_exporter( + DeploymentMode.O11Y, factory, project_id="explicit-project-id", log_stream_id="explicit-log-stream-id" + ) + + assert factory.calls[0]["headers"] == { + "X-SF-Token": "o11y-token", + "projectid": "explicit-project-id", + "logstreamid": "explicit-log-stream-id", + } + exporter.shutdown() + + +def test_exporter_copies_span_without_mutating_source() -> None: + factory = RecordingExporterFactory() + exporter = build_exporter( + DeploymentMode.O11Y, factory, project="authoritative-project", logstream="authoritative-log-stream" + ) + source = make_span() + source_resource = source.resource + source_attributes = dict(source.attributes or {}) + + assert exporter.export((source,)) == SpanExportResult.SUCCESS + + exported = factory.exporter.spans[0] + assert exported is not source + assert source.resource is source_resource + assert dict(source.attributes or {}) == source_attributes + assert source.resource.attributes["splunk_ao.project.name"] == "stale-resource-project" + assert source.attributes["splunk_ao.project.name"] == "stale-span-project" + assert exported.resource.attributes["splunk_ao.project.name"] == "authoritative-project" + assert exported.resource.attributes["splunk_ao.logstream.name"] == "authoritative-log-stream" + assert "splunk_ao.project.name" not in (exported.attributes or {}) + assert exported.resource.attributes["service.name"] == "checkout" + exporter.shutdown() + + +def test_exporter_preserves_every_unaffected_span_field() -> None: + factory = RecordingExporterFactory() + exporter = build_exporter(DeploymentMode.O11Y, factory, project="project") + source = make_span() + + exporter.export((source,)) + exported = factory.exporter.spans[0] + + for field in ( + "name", + "context", + "parent", + "events", + "links", + "kind", + "instrumentation_info", + "status", + "start_time", + "end_time", + "instrumentation_scope", + ): + assert getattr(exported, field) == getattr(source, field) + assert exported.resource.schema_url == source.resource.schema_url + assert exported.attributes["gen_ai.request.model"] == "gpt-4o" + assert exported.attributes["gen_ai.provider.name"] == "openai" + assert exported.attributes["gen_ai.system"] == "legacy-upstream-provider" + assert exported.attributes["custom.attribute"] == "preserved" + exporter.shutdown() + + +def test_exporter_delegates_force_flush_and_shutdown() -> None: + factory = RecordingExporterFactory() + exporter = build_exporter(DeploymentMode.O11Y, factory) + + assert exporter.force_flush(1234) is True + exporter.shutdown() + + assert factory.exporter.force_flush_timeouts == [1234] + assert factory.exporter.shutdown_calls == 1 + + +def test_processor_does_not_put_routing_on_span_attributes() -> None: + exporter = RecordingExporter() + processor = SplunkAOSpanProcessor( + project="project", logstream="log-stream", SpanProcessor=RecordingSpanProcessor, _exporter=exporter + ) + _project_context.set("later-project") + _log_stream_context.set("later-log-stream") + _experiment_id_context.set("later-experiment") + _session_id_context.set("session-id") + _dataset_input_context.set("question") + span = MagicMock() + + processor.on_start(span) + + calls = {args[0]: args[1] for args, _ in span.set_attribute.call_args_list} + assert not ROUTING_KEYS.intersection(calls) + assert calls["splunk_ao.session.id"] == "session-id" + assert calls["splunk_ao.dataset.input"] == "question" + processor.shutdown() + + +def test_processor_forwards_complete_routing_to_immutable_exporter() -> None: + exporter = RecordingExporter() + exporter_factory = MagicMock() + with patch("splunk_ao.otel.SplunkAOOTLPExporter", return_value=exporter) as exporter_class: + processor = SplunkAOSpanProcessor( + project_id="project-id", + log_stream_id="log-stream-id", + experiment_id="experiment-id", + SpanProcessor=RecordingSpanProcessor, + _exporter_factory=exporter_factory, + ) + + exporter_class.assert_called_once_with( + project=None, + project_id="project-id", + logstream=None, + log_stream_id="log-stream-id", + experiment_id="experiment-id", + _exporter_factory=exporter_factory, + ) + processor.shutdown() + + +def test_helper_constructs_registers_and_returns_processor() -> None: + provider = MagicMock() + resolved_processor = MagicMock(spec=SplunkAOSpanProcessor) + + with patch("splunk_ao.otel.SplunkAOSpanProcessor", return_value=resolved_processor) as processor_class: + result = add_splunk_ao_span_processor(provider, project="project") + + processor_class.assert_called_once_with(project="project") + provider.add_span_processor.assert_called_once_with(resolved_processor) + assert _TRACE_PROVIDER_CONTEXT_VAR.get() is provider + assert result is resolved_processor + + +def test_helper_accepts_prebuilt_processor() -> None: + provider = MagicMock() + processor = MagicMock(spec=SplunkAOSpanProcessor) + + assert add_splunk_ao_span_processor(provider, processor) is processor + provider.add_span_processor.assert_called_once_with(processor) + + +def test_helper_rejects_prebuilt_processor_and_constructor_kwargs() -> None: + provider = MagicMock() + processor = MagicMock(spec=SplunkAOSpanProcessor) + + with pytest.raises(ValueError, match="processor_kwargs"): + add_splunk_ao_span_processor(provider, processor, project="other") + provider.add_span_processor.assert_not_called() + + +def test_processor_construction_does_not_replace_global_provider() -> None: + global_before = trace.get_tracer_provider() + processor = SplunkAOSpanProcessor(SpanProcessor=RecordingSpanProcessor, _exporter=RecordingExporter()) + + assert trace.get_tracer_provider() is global_before + processor.shutdown() From a648ab62c776a803fb9f90648c0456c9fe52057b Mon Sep 17 00:00:00 2001 From: pradystar Date: Wed, 22 Jul 2026 07:13:05 -0700 Subject: [PATCH 02/10] feat(otel): add canonical attribute normalization --- splunk-ao-a2a/src/splunk_ao_a2a/_constants.py | 15 +- splunk-ao-a2a/src/splunk_ao_a2a/_context.py | 2 +- splunk-ao-a2a/src/splunk_ao_a2a/_spans.py | 16 +- splunk-ao-a2a/tests/test_spans.py | 11 +- .../tests/test_splunk_ao_compatibility.py | 9 +- src/splunk_ao/converter/__init__.py | 15 + src/splunk_ao/converter/attribute_mapping.py | 305 ++++++++++++++++++ src/splunk_ao/exporter/config.py | 13 +- src/splunk_ao/exporter/o11y.py | 14 +- src/splunk_ao/exporter/span_transform.py | 100 ++++++ src/splunk_ao/exporter/standalone.py | 14 +- src/splunk_ao/otel.py | 147 +-------- tests/test_attribute_mapping.py | 222 +++++++++++++ tests/test_exporter_config.py | 4 +- tests/test_exporter_o11y.py | 4 +- tests/test_otel.py | 204 ++++++------ tests/test_otel_native_paths.py | 5 +- tests/test_span_transform.py | 153 +++++++++ 18 files changed, 969 insertions(+), 284 deletions(-) create mode 100644 src/splunk_ao/converter/__init__.py create mode 100644 src/splunk_ao/converter/attribute_mapping.py create mode 100644 src/splunk_ao/exporter/span_transform.py create mode 100644 tests/test_attribute_mapping.py create mode 100644 tests/test_span_transform.py diff --git a/splunk-ao-a2a/src/splunk_ao_a2a/_constants.py b/splunk-ao-a2a/src/splunk_ao_a2a/_constants.py index 6dade0cb..efdb943d 100644 --- a/splunk-ao-a2a/src/splunk_ao_a2a/_constants.py +++ b/splunk-ao-a2a/src/splunk_ao_a2a/_constants.py @@ -10,16 +10,16 @@ SPLUNK_AO_OBSERVE_KEY = "splunk_ao_observe" AGNTCY_OBSERVE_KEY = "observe" # compatibility with AGNTCY Observe SDK -# A2A span attribute keys (match API-side A2A extension expectations) -A2A_TASK_ID = "a2a.task.id" -A2A_CONTEXT_ID = "a2a.context_id" -A2A_RPC_METHOD = "a2a.rpc.method" -A2A_TASK_STATE = "a2a.task.state" +# A2A span attribute keys +A2A_TASK_ID = "splunk_ao.a2a.task.id" +A2A_CONTEXT_ID = "splunk_ao.a2a.context_id" +A2A_RPC_METHOD = "splunk_ao.a2a.rpc.method" +A2A_TASK_STATE = "splunk_ao.a2a.task.state" # OTel GenAI semantic convention attributes — span type determination GENAI_OPERATION_NAME = "gen_ai.operation.name" GENAI_AGENT_NAME = "gen_ai.agent.name" -GENAI_SYSTEM = "gen_ai.system" +GENAI_CONVERSATION_ID = "gen_ai.conversation.id" GENAI_TOOL_NAME = "gen_ai.tool.name" # OTel GenAI semantic convention attributes — input/output content @@ -36,9 +36,6 @@ # Finish reasons FINISH_REASON_STOP = "stop" -# Session correlation -SESSION_ID = "session.id" - # Span link attributes for cross-agent correlation LINK_TYPE = "link.type" LINK_TYPE_AGENT_HANDOFF = "agent_handoff" diff --git a/splunk-ao-a2a/src/splunk_ao_a2a/_context.py b/splunk-ao-a2a/src/splunk_ao_a2a/_context.py index 12b9a1de..e938aa47 100644 --- a/splunk-ao-a2a/src/splunk_ao_a2a/_context.py +++ b/splunk-ao-a2a/src/splunk_ao_a2a/_context.py @@ -12,10 +12,10 @@ from splunk_ao_a2a._constants import ( AGNTCY_OBSERVE_KEY, - SPLUNK_AO_OBSERVE_KEY, LINK_FROM_AGENT, LINK_TYPE, LINK_TYPE_AGENT_HANDOFF, + SPLUNK_AO_OBSERVE_KEY, ) _logger = logging.getLogger(__name__) diff --git a/splunk-ao-a2a/src/splunk_ao_a2a/_spans.py b/splunk-ao-a2a/src/splunk_ao_a2a/_spans.py index 2e37f88c..ead1ac40 100644 --- a/splunk-ao-a2a/src/splunk_ao_a2a/_spans.py +++ b/splunk-ao-a2a/src/splunk_ao_a2a/_spans.py @@ -17,15 +17,14 @@ ERROR_STATES, FINISH_REASON_STOP, GENAI_AGENT_NAME, + GENAI_CONVERSATION_ID, GENAI_INPUT_MESSAGES, GENAI_OPERATION_NAME, GENAI_OUTPUT_MESSAGES, GENAI_RESPONSE_FINISH_REASONS, - GENAI_SYSTEM, GENAI_TOOL_NAME, ROLE_ASSISTANT, ROLE_USER, - SESSION_ID, ) @@ -68,7 +67,6 @@ def set_client_attributes( ) -> None: """Set standard A2A and GenAI attributes on a client span.""" span.set_attribute(GENAI_OPERATION_NAME, "invoke_agent") - span.set_attribute(GENAI_SYSTEM, "a2a") span.set_attribute(A2A_RPC_METHOD, rpc_method) if agent_name: @@ -76,7 +74,7 @@ def set_client_attributes( if hasattr(request, "context_id") and request.context_id: span.set_attribute(A2A_CONTEXT_ID, str(request.context_id)) - span.set_attribute(SESSION_ID, str(request.context_id)) + span.set_attribute(GENAI_CONVERSATION_ID, str(request.context_id)) if hasattr(request, "task_id") and request.task_id: span.set_attribute(A2A_TASK_ID, str(request.task_id)) @@ -90,7 +88,6 @@ def set_server_attributes( ) -> None: """Set standard A2A and GenAI attributes on a server span.""" span.set_attribute(GENAI_OPERATION_NAME, "invoke_agent") - span.set_attribute(GENAI_SYSTEM, "a2a") span.set_attribute(A2A_RPC_METHOD, rpc_method) if agent_name: @@ -101,7 +98,7 @@ def set_server_attributes( context_id = getattr(message, "context_id", None) if context_id: span.set_attribute(A2A_CONTEXT_ID, str(context_id)) - span.set_attribute(SESSION_ID, str(context_id)) + span.set_attribute(GENAI_CONVERSATION_ID, str(context_id)) task_id = getattr(message, "task_id", None) if task_id: @@ -111,7 +108,6 @@ def set_server_attributes( def set_tool_attributes(span: trace.Span, rpc_method: str) -> None: """Set attributes for tool-like A2A operations (get_task, cancel_task, get_card).""" span.set_attribute(GENAI_OPERATION_NAME, "execute_tool") - span.set_attribute(GENAI_SYSTEM, "a2a") span.set_attribute(A2A_RPC_METHOD, rpc_method) span.set_attribute(GENAI_TOOL_NAME, rpc_method) @@ -180,7 +176,7 @@ def set_simple_input(span: trace.Span, args: tuple, rpc_method: str) -> None: def track_task_state(span: trace.Span, obj: Any) -> None: """Record A2A task state and ID on *span*. - Sets ``a2a.task.state``, ``a2a.task.id``, and ``gen_ai.response.finish_reasons``. + Sets the A2A task state and ID plus ``gen_ai.response.finish_reasons``. Marks the span as an error when the task enters a terminal error state. """ if obj is None: @@ -195,9 +191,9 @@ def track_task_state(span: trace.Span, obj: Any) -> None: if state_value in ERROR_STATES: span.set_status(StatusCode.ERROR, f"A2A task {state_value}") - span.set_attribute(GENAI_RESPONSE_FINISH_REASONS, json.dumps([state_value])) + span.set_attribute(GENAI_RESPONSE_FINISH_REASONS, (state_value,)) elif state_value == "completed": - span.set_attribute(GENAI_RESPONSE_FINISH_REASONS, json.dumps([FINISH_REASON_STOP])) + span.set_attribute(GENAI_RESPONSE_FINISH_REASONS, (FINISH_REASON_STOP,)) task_id = getattr(obj, "id", None) if task_id: diff --git a/splunk-ao-a2a/tests/test_spans.py b/splunk-ao-a2a/tests/test_spans.py index 39e4f196..7b410d80 100644 --- a/splunk-ao-a2a/tests/test_spans.py +++ b/splunk-ao-a2a/tests/test_spans.py @@ -14,13 +14,12 @@ A2A_TASK_ID, A2A_TASK_STATE, GENAI_AGENT_NAME, + GENAI_CONVERSATION_ID, GENAI_INPUT_MESSAGES, GENAI_OPERATION_NAME, GENAI_OUTPUT_MESSAGES, GENAI_RESPONSE_FINISH_REASONS, - GENAI_SYSTEM, GENAI_TOOL_NAME, - SESSION_ID, ) @@ -55,12 +54,12 @@ def test_sets_standard_attributes(self, span): # Then: all expected attributes are set span.set_attribute.assert_any_call(GENAI_OPERATION_NAME, "invoke_agent") - span.set_attribute.assert_any_call(GENAI_SYSTEM, "a2a") span.set_attribute.assert_any_call(A2A_RPC_METHOD, "SendMessage") span.set_attribute.assert_any_call(GENAI_AGENT_NAME, "my-agent") span.set_attribute.assert_any_call(A2A_CONTEXT_ID, "ctx-1") - span.set_attribute.assert_any_call(SESSION_ID, "ctx-1") + span.set_attribute.assert_any_call(GENAI_CONVERSATION_ID, "ctx-1") span.set_attribute.assert_any_call(A2A_TASK_ID, "task-1") + assert "gen_ai.system" not in [call.args[0] for call in span.set_attribute.call_args_list] def test_skips_agent_name_when_none(self, span): # Given: no agent name @@ -84,7 +83,7 @@ def test_sets_attributes_from_message(self, span): # Then: attributes extracted from message span.set_attribute.assert_any_call(A2A_CONTEXT_ID, "ctx-2") - span.set_attribute.assert_any_call(SESSION_ID, "ctx-2") + span.set_attribute.assert_any_call(GENAI_CONVERSATION_ID, "ctx-2") span.set_attribute.assert_any_call(A2A_TASK_ID, "task-2") @@ -183,7 +182,7 @@ def test_sets_completed_state(self, span): # Then: state and finish reason set span.set_attribute.assert_any_call(A2A_TASK_STATE, "completed") - span.set_attribute.assert_any_call(GENAI_RESPONSE_FINISH_REASONS, json.dumps(["stop"])) + span.set_attribute.assert_any_call(GENAI_RESPONSE_FINISH_REASONS, ("stop",)) span.set_attribute.assert_any_call(A2A_TASK_ID, "t-1") @pytest.mark.parametrize("state", ["failed", "rejected", "canceled"]) diff --git a/splunk-ao-a2a/tests/test_splunk_ao_compatibility.py b/splunk-ao-a2a/tests/test_splunk_ao_compatibility.py index 3913db80..330078b9 100644 --- a/splunk-ao-a2a/tests/test_splunk_ao_compatibility.py +++ b/splunk-ao-a2a/tests/test_splunk_ao_compatibility.py @@ -73,8 +73,13 @@ def exporter_factory(**kwargs: object) -> RecordingExporter: } exported = delegate.spans[0] assert exported.instrumentation_scope.name == INSTRUMENTOR_NAME - assert exported.attributes["gen_ai.system"] == "a2a" - assert exported.attributes["a2a.rpc.method"] == "SendMessage" + assert "gen_ai.system" not in exported.attributes + assert exported.attributes["splunk_ao.system"] == "splunk_ao_python" + assert exported.attributes["splunk_ao.a2a.rpc.method"] == "SendMessage" + assert exported.attributes["gen_ai.conversation.id"] == "context-id" + assert exported.attributes["splunk_ao.session.id"] == "context-id" + assert exported.attributes["gen_ai.operation.name"] == "invoke_agent" + assert exported.attributes["splunk_ao.operation.name"] == "invoke_agent" assert exported.resource.attributes["splunk_ao.project.name"] == "a2a-project" assert exported.resource.attributes["splunk_ao.logstream.name"] == "a2a-log-stream" assert "splunk_ao.project.name" not in exported.attributes diff --git a/src/splunk_ao/converter/__init__.py b/src/splunk_ao/converter/__init__.py new file mode 100644 index 00000000..654ae5dd --- /dev/null +++ b/src/splunk_ao/converter/__init__.py @@ -0,0 +1,15 @@ +"""Conversion helpers shared by Splunk AO telemetry paths.""" + +from splunk_ao.converter.attribute_mapping import ( + CONTENT_ALIAS_BY_GEN_AI, + SPLUNK_ALIAS_BY_GEN_AI, + build_span_attributes, + normalize_attributes_for_export, +) + +__all__ = [ + "CONTENT_ALIAS_BY_GEN_AI", + "SPLUNK_ALIAS_BY_GEN_AI", + "build_span_attributes", + "normalize_attributes_for_export", +] diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py new file mode 100644 index 00000000..74d60ceb --- /dev/null +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -0,0 +1,305 @@ +"""Canonical Galileo-field and OTLP wire-attribute mapping.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, MutableMapping, Sequence +from enum import Enum +from typing import Any + +from opentelemetry.util.types import AttributeValue +from pydantic import BaseModel + +from galileo_core.schemas.logging.span import AgentSpan, LlmSpan, RetrieverSpan, ToolSpan, WorkflowSpan +from galileo_core.schemas.logging.step import BaseStep, StepType + +SPLUNK_AO_SYSTEM = "splunk_ao.system" +SPLUNK_AO_SYSTEM_VALUE = "splunk_ao_python" + +CONTENT_ALIAS_BY_GEN_AI: Mapping[str, str] = { + "gen_ai.input.messages": "splunk_ao.input.messages", + "gen_ai.system_instructions": "splunk_ao.system_instructions", + "gen_ai.output.messages": "splunk_ao.output.messages", + "gen_ai.tool.call.arguments": "splunk_ao.tool.call.arguments", + "gen_ai.tool.call.result": "splunk_ao.tool.call.result", + "gen_ai.retrieval.documents": "splunk_ao.retrieval.documents", + "gen_ai.tool.definitions": "splunk_ao.tool.definitions", +} + +SPLUNK_ALIAS_BY_GEN_AI: Mapping[str, str] = { + "gen_ai.system": "splunk_ao.provider.name", + "gen_ai.operation.name": "splunk_ao.operation.name", + "gen_ai.conversation.id": "splunk_ao.session.id", + "gen_ai.workflow.name": "splunk_ao.workflow.name", + "gen_ai.agent.name": "splunk_ao.agent.name", + "gen_ai.agent.id": "splunk_ao.agent.id", + "gen_ai.agent.description": "splunk_ao.agent.description", + "gen_ai.agent.version": "splunk_ao.agent.version", + "gen_ai.provider.name": "splunk_ao.provider.name", + "gen_ai.request.model": "splunk_ao.request.model", + "gen_ai.request.temperature": "splunk_ao.request.temperature", + "gen_ai.request.top_p": "splunk_ao.request.top_p", + "gen_ai.request.top_k": "splunk_ao.request.top_k", + "gen_ai.request.max_tokens": "splunk_ao.request.max_tokens", + "gen_ai.request.stop_sequences": "splunk_ao.request.stop_sequences", + "gen_ai.request.frequency_penalty": "splunk_ao.request.frequency_penalty", + "gen_ai.request.presence_penalty": "splunk_ao.request.presence_penalty", + "gen_ai.request.seed": "splunk_ao.request.seed", + "gen_ai.response.finish_reasons": "splunk_ao.response.finish_reasons", + "gen_ai.response.model": "splunk_ao.response.model", + "gen_ai.response.id": "splunk_ao.response.id", + "gen_ai.output.type": "splunk_ao.output.type", + "gen_ai.tool.call.id": "splunk_ao.tool.call.id", + "gen_ai.tool.name": "splunk_ao.tool.name", + "gen_ai.tool.description": "splunk_ao.tool.description", + "gen_ai.tool.type": "splunk_ao.tool.type", + "gen_ai.retrieval.top_k": "splunk_ao.retrieval.top_k", + "gen_ai.retrieval.query.text": "splunk_ao.retrieval.query.text", + "gen_ai.usage.input_tokens": "splunk_ao.llm.usage.input_tokens", + "gen_ai.usage.output_tokens": "splunk_ao.llm.usage.output_tokens", + "gen_ai.usage.cache_creation.input_tokens": "splunk_ao.llm.usage.cache_creation.input_tokens", + "gen_ai.usage.cache_read.input_tokens": "splunk_ao.llm.usage.cache_read.input_tokens", + "gen_ai.usage.reasoning.output_tokens": "splunk_ao.llm.usage.reasoning.output_tokens", + "gen_ai.response.time_to_first_chunk": "splunk_ao.llm.time_to_first_token_ns", + **CONTENT_ALIAS_BY_GEN_AI, +} + +_OPERATION_BY_STEP_TYPE = { + StepType.llm: "chat", + StepType.retriever: "retrieval", + StepType.tool: "execute_tool", + StepType.workflow: "invoke_workflow", + StepType.agent: "invoke_agent", +} + + +def _json_compatible(value: Any) -> Any: + if isinstance(value, BaseModel): + return value.model_dump(mode="json", exclude_none=True) + if isinstance(value, Enum): + return value.value + if isinstance(value, Mapping): + return {str(key): _json_compatible(item) for key, item in value.items()} + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + return [_json_compatible(item) for item in value] + return value + + +def _json_string(value: Any) -> str: + return json.dumps(_json_compatible(value), separators=(",", ":"), sort_keys=True, default=str) + + +def _content_value(value: Any) -> str: + return value if isinstance(value, str) else _json_string(value) + + +def _set_if_present(attrs: MutableMapping[str, AttributeValue], key: str, value: AttributeValue | None) -> None: + if value is not None: + attrs[key] = value + + +def _field(source: Any, name: str) -> Any: + value = getattr(source, name, None) + if value is not None: + return value + extra = getattr(source, "model_extra", None) + return extra.get(name) if extra else None + + +def set_common_attributes( + attrs: MutableMapping[str, AttributeValue], span: BaseStep, session_id: str | None = None +) -> None: + """Map fields shared by every proprietary span type.""" + if span.redacted_input is not None: + attrs["splunk_ao.redacted_input"] = _content_value(span.redacted_input) + if span.redacted_output is not None: + attrs["splunk_ao.redacted_output"] = _content_value(span.redacted_output) + if span.user_metadata: + attrs["splunk_ao.metadata"] = _json_string(span.user_metadata) + if span.tags: + attrs["splunk_ao.tags"] = tuple(span.tags) + if span.status_code is not None: + attrs["splunk_ao.status_code"] = span.status_code + if span.status_code >= 400: + attrs["error.type"] = str(span.status_code) + + step_number = getattr(span, "step_number", None) + if step_number is not None: + attrs["splunk_ao.step_number"] = step_number + + resolved_session_id = session_id or getattr(span, "session_id", None) + if resolved_session_id is not None: + attrs["gen_ai.conversation.id"] = str(resolved_session_id) + + +def set_dataset_attributes(attrs: MutableMapping[str, AttributeValue], span: BaseStep) -> None: + """Map optional dataset context fields.""" + _set_if_present(attrs, "splunk_ao.dataset.input", span.dataset_input) + _set_if_present(attrs, "splunk_ao.dataset.output", span.dataset_output) + if span.dataset_metadata: + attrs["splunk_ao.dataset.metadata"] = _json_string(span.dataset_metadata) + + +def _set_operation(attrs: MutableMapping[str, AttributeValue], span_type: StepType) -> None: + operation = _OPERATION_BY_STEP_TYPE.get(span_type) + if operation is not None: + attrs["gen_ai.operation.name"] = operation + + +def set_llm_attributes(attrs: MutableMapping[str, AttributeValue], span: LlmSpan) -> None: + """Map LLM request, response, content, and metric fields.""" + _set_operation(attrs, StepType.llm) + attrs["gen_ai.input.messages"] = _content_value(span.input) + attrs["gen_ai.output.messages"] = _content_value(span.output) + _set_if_present(attrs, "gen_ai.request.model", span.model) + _set_if_present(attrs, "gen_ai.request.temperature", span.temperature) + + for field_name, attribute_name in ( + ("provider", "gen_ai.provider.name"), + ("top_p", "gen_ai.request.top_p"), + ("top_k", "gen_ai.request.top_k"), + ("max_tokens", "gen_ai.request.max_tokens"), + ("stop_sequences", "gen_ai.request.stop_sequences"), + ("frequency_penalty", "gen_ai.request.frequency_penalty"), + ("presence_penalty", "gen_ai.request.presence_penalty"), + ("seed", "gen_ai.request.seed"), + ("response_model", "gen_ai.response.model"), + ("response_id", "gen_ai.response.id"), + ): + _set_if_present(attrs, attribute_name, _field(span, field_name)) + + if span.finish_reason is not None: + attrs["gen_ai.response.finish_reasons"] = (span.finish_reason,) + if span.tools is not None: + attrs["gen_ai.tool.definitions"] = _json_string(span.tools) + if span.events is not None: + attrs["splunk_ao.llm.events"] = _json_string(span.events) + + metrics = span.metrics + _set_if_present(attrs, "gen_ai.usage.input_tokens", metrics.num_input_tokens) + _set_if_present(attrs, "gen_ai.usage.output_tokens", metrics.num_output_tokens) + _set_if_present(attrs, "splunk_ao.llm.usage.total_tokens", metrics.num_total_tokens) + if metrics.time_to_first_token_ns is not None: + attrs["gen_ai.response.time_to_first_chunk"] = metrics.time_to_first_token_ns / 1_000_000_000 + attrs["splunk_ao.llm.time_to_first_token_ns"] = metrics.time_to_first_token_ns + + for field_name, attribute_name in ( + ("input_cost", "splunk_ao.llm.cost.input_usd"), + ("output_cost", "splunk_ao.llm.cost.output_usd"), + ("total_cost", "splunk_ao.llm.cost.total_usd"), + ("cache_creation_input_tokens", "gen_ai.usage.cache_creation.input_tokens"), + ("cache_read_input_tokens", "gen_ai.usage.cache_read.input_tokens"), + ("reasoning_output_tokens", "gen_ai.usage.reasoning.output_tokens"), + ): + _set_if_present(attrs, attribute_name, _field(metrics, field_name)) + + for field_name, attribute_name in ( + ("log_probs", "splunk_ao.llm.log_probs"), + ("top_logprobs", "splunk_ao.llm.top_logprobs"), + ("response_format", "splunk_ao.llm.response_format"), + ("tool_use_allowed", "splunk_ao.llm.tool_use_allowed"), + ("structured_output_name", "splunk_ao.llm.structured_output.name"), + ("structured_output_input", "splunk_ao.llm.structured_output.input"), + ): + _set_if_present(attrs, attribute_name, _field(span, field_name)) + + +def set_tool_attributes(attrs: MutableMapping[str, AttributeValue], span: ToolSpan) -> None: + """Map tool execution fields.""" + _set_operation(attrs, StepType.tool) + attrs["gen_ai.tool.name"] = span.name + attrs["gen_ai.tool.call.arguments"] = _content_value(span.input) + if span.output is not None: + attrs["gen_ai.tool.call.result"] = _content_value(span.output) + _set_if_present(attrs, "gen_ai.tool.call.id", span.tool_call_id) + + +def set_retriever_attributes(attrs: MutableMapping[str, AttributeValue], span: RetrieverSpan) -> None: + """Map retrieval query and document fields.""" + _set_operation(attrs, StepType.retriever) + attrs["gen_ai.retrieval.query.text"] = span.input + attrs["gen_ai.retrieval.documents"] = _content_value(span.output) + attrs["splunk_ao.retrieval.documents.count"] = len(span.output) + attrs["db.operation"] = "search" + requested_top_k = _field(span, "num_documents") + _set_if_present(attrs, "gen_ai.retrieval.top_k", requested_top_k) + + +def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: WorkflowSpan | AgentSpan) -> None: + attrs["gen_ai.input.messages"] = _content_value(span.input) + if span.output is not None: + attrs["gen_ai.output.messages"] = _content_value(span.output) + + +def set_workflow_attributes(attrs: MutableMapping[str, AttributeValue], span: WorkflowSpan) -> None: + """Map workflow fields.""" + _set_operation(attrs, StepType.workflow) + attrs["gen_ai.workflow.name"] = span.name + _set_orchestration_content(attrs, span) + + +def set_agent_attributes(attrs: MutableMapping[str, AttributeValue], span: AgentSpan) -> None: + """Map agent fields.""" + _set_operation(attrs, StepType.agent) + attrs["gen_ai.agent.name"] = span.name + attrs["splunk_ao.agent.type"] = span.agent_type.value + _set_orchestration_content(attrs, span) + + +def _set_generic_content(attrs: MutableMapping[str, AttributeValue], span: BaseStep) -> None: + if span.input is not None: + attrs["gen_ai.input.messages"] = _content_value(span.input) + if span.output is not None: + attrs["gen_ai.output.messages"] = _content_value(span.output) + + +def build_span_attributes(span: BaseStep, session_id: str | None = None) -> dict[str, AttributeValue]: + """Build preliminary attributes for one proprietary Galileo span.""" + attrs: dict[str, AttributeValue] = {} + set_common_attributes(attrs, span, session_id) + set_dataset_attributes(attrs, span) + + if isinstance(span, LlmSpan): + set_llm_attributes(attrs, span) + elif isinstance(span, ToolSpan): + set_tool_attributes(attrs, span) + elif isinstance(span, RetrieverSpan): + set_retriever_attributes(attrs, span) + elif isinstance(span, AgentSpan): + set_agent_attributes(attrs, span) + elif isinstance(span, WorkflowSpan): + set_workflow_attributes(attrs, span) + elif getattr(span.type, "value", span.type) == "control": + attrs["splunk_ao.operation.name"] = "control" + _set_generic_content(attrs, span) + elif span.type == StepType.trace: + _set_generic_content(attrs, span) + else: + raise TypeError(f"Unsupported span type: {type(span).__name__}") + + return attrs + + +def _alias_value(source_key: str, value: AttributeValue) -> AttributeValue: + if source_key == "gen_ai.response.time_to_first_chunk": + return int(float(value) * 1_000_000_000) + return value + + +def normalize_attributes_for_export( + attrs: Mapping[str, AttributeValue], *, enabled: bool = True +) -> dict[str, AttributeValue]: + """Return final wire attributes without mutating the source mapping.""" + result = dict(attrs) + if not enabled: + return result + + for source_key, destination_key in SPLUNK_ALIAS_BY_GEN_AI.items(): + if source_key in attrs: + result[destination_key] = _alias_value(source_key, attrs[source_key]) + + for source_key in CONTENT_ALIAS_BY_GEN_AI: + result.pop(source_key, None) + + result[SPLUNK_AO_SYSTEM] = SPLUNK_AO_SYSTEM_VALUE + return result diff --git a/src/splunk_ao/exporter/config.py b/src/splunk_ao/exporter/config.py index 176165da..2c282bc7 100644 --- a/src/splunk_ao/exporter/config.py +++ b/src/splunk_ao/exporter/config.py @@ -2,8 +2,13 @@ from collections.abc import Callable from dataclasses import dataclass +from typing import Any from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace.export import SpanExporter + +from splunk_ao.exporter.span_transform import NormalizingSpanExporter @dataclass @@ -25,7 +30,7 @@ class RoutingAttrs: experiment_id: str | None = None -ExporterFactory = Callable[..., OTLPSpanExporter] +ExporterFactory = Callable[..., SpanExporter] def resolve_exporter_config(endpoint: str, auth_header: tuple[str, str], routing: RoutingAttrs) -> ExporterConfig: @@ -69,7 +74,9 @@ def build_exporter( auth_header: tuple[str, str], routing: RoutingAttrs, _exporter_factory: ExporterFactory = OTLPSpanExporter, -) -> OTLPSpanExporter: + **exporter_kwargs: Any, +) -> SpanExporter: """Build an OTLP HTTP exporter from shared resolved configuration.""" config = resolve_exporter_config(endpoint, auth_header, routing) - return _exporter_factory(endpoint=config.endpoint, headers=config.headers) + delegate = _exporter_factory(endpoint=config.endpoint, headers=config.headers, **exporter_kwargs) + return NormalizingSpanExporter(delegate, Resource(routing_resource_attributes(routing))) diff --git a/src/splunk_ao/exporter/o11y.py b/src/splunk_ao/exporter/o11y.py index d6584770..69b00ec5 100644 --- a/src/splunk_ao/exporter/o11y.py +++ b/src/splunk_ao/exporter/o11y.py @@ -1,6 +1,9 @@ """Splunk Observability Cloud OTLP exporter construction.""" +from typing import Any + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace.export import SpanExporter from splunk_ao.deployment import O11yConfig from splunk_ao.exporter.config import ( @@ -22,7 +25,12 @@ def resolve_o11y_exporter_config(config: O11yConfig, routing: RoutingAttrs) -> E def build_o11y_exporter( - config: O11yConfig, routing: RoutingAttrs, _exporter_factory: ExporterFactory = OTLPSpanExporter -) -> OTLPSpanExporter: + config: O11yConfig, + routing: RoutingAttrs, + _exporter_factory: ExporterFactory = OTLPSpanExporter, + **exporter_kwargs: Any, +) -> SpanExporter: """Build an OTLP exporter authenticated for Splunk Observability Cloud.""" - return build_exporter(config.otlp_endpoint, _o11y_auth_header(config), routing, _exporter_factory) + return build_exporter( + config.otlp_endpoint, _o11y_auth_header(config), routing, _exporter_factory, **exporter_kwargs + ) diff --git a/src/splunk_ao/exporter/span_transform.py b/src/splunk_ao/exporter/span_transform.py new file mode 100644 index 00000000..76d042f9 --- /dev/null +++ b/src/splunk_ao/exporter/span_transform.py @@ -0,0 +1,100 @@ +"""Immutable OTLP span normalization before serialization.""" + +from __future__ import annotations + +import os +from collections.abc import Sequence + +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult + +from splunk_ao.converter.attribute_mapping import normalize_attributes_for_export + +ROUTING_ATTRIBUTE_KEYS = frozenset( + { + "splunk_ao.project.name", + "splunk_ao.project.id", + "splunk_ao.logstream.name", + "splunk_ao.logstream.id", + "splunk_ao.experiment.id", + } +) + +_NORMALIZATION_ENV = "SPLUNK_AO_DEV_ENABLE_ATTRIBUTE_NORMALIZATION" +_FALSE_VALUES = frozenset({"0", "false", "no", "off"}) + + +def _normalization_enabled() -> bool: + value = os.environ.get(_NORMALIZATION_ENV) + return value is None or value.strip().lower() not in _FALSE_VALUES + + +def copy_span_for_export( + span: ReadableSpan, routing_resource: Resource | None = None, *, normalize_attributes: bool = True +) -> ReadableSpan: + """Return an immutable span copy with final attributes and routing.""" + source_attributes = { + key: value for key, value in (span.attributes or {}).items() if key not in ROUTING_ATTRIBUTE_KEYS + } + attributes = normalize_attributes_for_export(source_attributes, enabled=normalize_attributes) + + source_resource = span.resource or Resource({}) + base_resource = Resource( + {key: value for key, value in source_resource.attributes.items() if key not in ROUTING_ATTRIBUTE_KEYS}, + schema_url=source_resource.schema_url, + ) + resource = base_resource.merge(routing_resource) if routing_resource is not None else base_resource + + return ReadableSpan( + name=span.name, + context=span.context, + parent=span.parent, + resource=resource, + attributes=attributes, + events=span.events, + links=span.links, + kind=span.kind, + instrumentation_info=span.instrumentation_info, + status=span.status, + start_time=span.start_time, + end_time=span.end_time, + instrumentation_scope=span.instrumentation_scope, + ) + + +class NormalizingSpanExporter(SpanExporter): + """Apply canonical attributes and routing once before delegating export.""" + + def __init__( + self, + delegate: SpanExporter, + routing_resource: Resource | None = None, + *, + normalize_attributes: bool | None = None, + ) -> None: + self._delegate = delegate + self._routing_resource = routing_resource + self._normalize_attributes = _normalization_enabled() if normalize_attributes is None else normalize_attributes + + @property + def delegate(self) -> SpanExporter: + """Return the wrapped exporter for SDK composition and diagnostics.""" + return self._delegate + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + """Export normalized immutable copies of the supplied spans.""" + return self._delegate.export( + tuple( + copy_span_for_export(span, self._routing_resource, normalize_attributes=self._normalize_attributes) + for span in spans + ) + ) + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Delegate flushing exactly once.""" + return self._delegate.force_flush(timeout_millis) + + def shutdown(self) -> None: + """Delegate shutdown exactly once.""" + self._delegate.shutdown() diff --git a/src/splunk_ao/exporter/standalone.py b/src/splunk_ao/exporter/standalone.py index a1034aed..0aab409c 100644 --- a/src/splunk_ao/exporter/standalone.py +++ b/src/splunk_ao/exporter/standalone.py @@ -1,6 +1,9 @@ """Standalone Splunk AO OTLP exporter construction.""" +from typing import Any + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace.export import SpanExporter from splunk_ao.deployment import StandaloneConfig from splunk_ao.exporter.config import ( @@ -22,7 +25,12 @@ def resolve_standalone_exporter_config(config: StandaloneConfig, routing: Routin def build_standalone_exporter( - config: StandaloneConfig, routing: RoutingAttrs, _exporter_factory: ExporterFactory = OTLPSpanExporter -) -> OTLPSpanExporter: + config: StandaloneConfig, + routing: RoutingAttrs, + _exporter_factory: ExporterFactory = OTLPSpanExporter, + **exporter_kwargs: Any, +) -> SpanExporter: """Build an OTLP exporter authenticated for standalone Splunk AO.""" - return build_exporter(config.otlp_endpoint, _standalone_auth_header(config), routing, _exporter_factory) + return build_exporter( + config.otlp_endpoint, _standalone_auth_header(config), routing, _exporter_factory, **exporter_kwargs + ) diff --git a/src/splunk_ao/otel.py b/src/splunk_ao/otel.py index 9e5ff2bc..3c81ff94 100644 --- a/src/splunk_ao/otel.py +++ b/src/splunk_ao/otel.py @@ -9,14 +9,13 @@ from opentelemetry import context, trace from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult from opentelemetry.trace import Tracer -from galileo_core.schemas.logging.span import RetrieverSpan, ToolSpan, WorkflowSpan from galileo_core.schemas.logging.span import Span as GalileoSpan from splunk_ao.config import SplunkAOConfig +from splunk_ao.converter import build_span_attributes from splunk_ao.decorator import ( _dataset_input_context, _dataset_metadata_context, @@ -27,12 +26,7 @@ _session_id_context, ) from splunk_ao.deployment import DeploymentMode, O11yConfig, StandaloneConfig -from splunk_ao.exporter import ( - RoutingAttrs, - resolve_o11y_exporter_config, - resolve_standalone_exporter_config, - routing_resource_attributes, -) +from splunk_ao.exporter import RoutingAttrs, build_o11y_exporter, build_standalone_exporter from splunk_ao.utils.env_helpers import ( _get_log_stream_from_env, _get_log_stream_id_from_env, @@ -41,7 +35,6 @@ _get_project_id_from_env, _get_project_or_default, ) -from splunk_ao.utils.retrievers import document_adapter logger = logging.getLogger(__name__) @@ -61,17 +54,6 @@ def get_tracer( _TRACE_PROVIDER_CONTEXT_VAR: ContextVar[TracerProvider | None] = ContextVar("galileo_trace_provider", default=None) -ROUTING_ATTRIBUTE_KEYS = frozenset( - { - "splunk_ao.project.name", - "splunk_ao.project.id", - "splunk_ao.logstream.name", - "splunk_ao.logstream.id", - "splunk_ao.experiment.id", - } -) - - def _resolve_name_or_id( explicit_name: str | None, explicit_id: str | None, @@ -129,31 +111,6 @@ def _resolve_routing( ) -def _with_routing_resource(span: ReadableSpan, routing_resource: Resource) -> ReadableSpan: - """Return an immutable copy with authoritative routing in its Resource.""" - attributes = {key: value for key, value in (span.attributes or {}).items() if key not in ROUTING_ATTRIBUTE_KEYS} - source_resource = span.resource or Resource({}) - base_resource = Resource( - {key: value for key, value in source_resource.attributes.items() if key not in ROUTING_ATTRIBUTE_KEYS}, - schema_url=source_resource.schema_url, - ) - return ReadableSpan( - name=span.name, - context=span.context, - parent=span.parent, - resource=base_resource.merge(routing_resource), - attributes=attributes, - events=span.events, - links=span.links, - kind=span.kind, - instrumentation_info=span.instrumentation_info, - status=span.status, - start_time=span.start_time, - end_time=span.end_time, - instrumentation_scope=span.instrumentation_scope, - ) - - class SplunkAOOTLPExporter(SpanExporter): """ OpenTelemetry OTLP span exporter preconfigured for Splunk AO. @@ -197,21 +154,21 @@ def __init__( deployment = config.resolve_deployment() self._routing = _resolve_routing(deployment, project, project_id, logstream, log_stream_id, experiment_id) if deployment == DeploymentMode.O11Y: - exporter_config = resolve_o11y_exporter_config(O11yConfig.from_env(), self._routing) + self._delegate = build_o11y_exporter(O11yConfig.from_env(), self._routing, _exporter_factory, **kwargs) else: - exporter_config = resolve_standalone_exporter_config(StandaloneConfig.from_env(), self._routing) + self._delegate = build_standalone_exporter( + StandaloneConfig.from_env(), self._routing, _exporter_factory, **kwargs + ) self.project = self._routing.project_name self.project_id = self._routing.project_id self.logstream = self._routing.log_stream_name self.log_stream_id = self._routing.log_stream_id self.experiment_id = self._routing.experiment_id - self._routing_resource = Resource(routing_resource_attributes(self._routing)) - self._delegate = _exporter_factory(endpoint=exporter_config.endpoint, headers=exporter_config.headers, **kwargs) def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: - """Export immutable copies with authoritative routing Resources.""" - return self._delegate.export(tuple(_with_routing_resource(span, self._routing_resource) for span in spans)) + """Export through the shared immutable normalization pipeline.""" + return self._delegate.export(spans) def force_flush(self, timeout_millis: int = 30000) -> bool: """Flush the delegate exporter.""" @@ -291,7 +248,7 @@ def on_start(self, span: Span, parent_context: context.Context | None = None) -> session_id = _session_id_context.get(None) if session_id: - span.set_attribute("splunk_ao.session.id", session_id) + span.set_attribute("gen_ai.conversation.id", session_id) # Set dataset attributes for ground truth/reference output support _apply_dataset_attributes( @@ -340,34 +297,6 @@ def add_splunk_ao_span_processor( return resolved_processor -def _set_retriever_span_attributes(span: trace.Span, galileo_span: RetrieverSpan) -> None: - span.set_attribute("db.operation", "search") - span.set_attribute("gen_ai.input.messages", json.dumps([{"role": "user", "content": galileo_span.input}])) - span.set_attribute( - "gen_ai.output.messages", - json.dumps( - [ - { - "role": "assistant", - "content": {"documents": document_adapter.dump_python(galileo_span.output, mode="json")}, - } - ] - ), - ) - - -def _set_tool_span_attributes(span: trace.Span, galileo_span: ToolSpan) -> None: - span.set_attribute("gen_ai.operation.name", "execute_tool") - span.set_attribute("gen_ai.tool.name", galileo_span.name) - span.set_attribute("gen_ai.tool.call.arguments", galileo_span.input) - span.set_attribute("gen_ai.input.messages", json.dumps([{"role": "tool", "content": galileo_span.input}])) - if galileo_span.output is not None: - span.set_attribute("gen_ai.tool.call.result", galileo_span.output) - span.set_attribute("gen_ai.output.messages", json.dumps([{"role": "tool", "content": galileo_span.output}])) - if galileo_span.tool_call_id is not None: - span.set_attribute("gen_ai.tool.call.id", galileo_span.tool_call_id) - - def _apply_dataset_attributes( span: trace.Span, dataset_input: str | None, dataset_output: str | None, dataset_metadata: dict[str, Any] | None ) -> None: @@ -380,48 +309,6 @@ def _apply_dataset_attributes( span.set_attribute("splunk_ao.dataset.metadata", json.dumps(dataset_metadata)) -def _set_workflow_span_attributes(span: trace.Span, galileo_span: WorkflowSpan) -> None: - """Set OpenTelemetry attributes for WorkflowSpan.""" - # Handle input - Union[str, Sequence[Message]] - if isinstance(galileo_span.input, str): - input_messages = [{"role": "user", "content": galileo_span.input}] - else: - # Sequence[Message] - serialize each message - input_messages = [] - for msg in list(galileo_span.input): - if hasattr(msg, "model_dump"): - input_messages.append(msg.model_dump(exclude_none=True)) - else: - input_messages.append(msg) - span.set_attribute("gen_ai.input.messages", json.dumps(input_messages)) - - # Handle output - Union[str, Message, Sequence[Document], None] - if galileo_span.output is None: - return - - output_value = galileo_span.output - # Type annotation to handle flexible content types (string or dict) - # Content can be: str (simple output), dict (documents), or dict (Message model_dump) - output_messages: list[dict[str, Any]] = [] - - if isinstance(output_value, str): - output_messages = [{"role": "assistant", "content": output_value}] - elif hasattr(output_value, "model_dump"): - # Single Message - output_messages = [output_value.model_dump(exclude_none=True)] - else: - # Sequence[Document] - wrap in assistant message - # Use document_adapter for consistency with _set_retriever_span_attributes - output_messages = [ - { - "role": "assistant", - "content": {"documents": document_adapter.dump_python(list(output_value), mode="json")}, - } - ] - - span.set_attribute("gen_ai.output.messages", json.dumps(output_messages)) - - @contextmanager def start_splunk_ao_span(galileo_span: GalileoSpan) -> Generator[trace.Span, Any, None]: tracer_provider = _TRACE_PROVIDER_CONTEXT_VAR.get() @@ -430,14 +317,8 @@ def start_splunk_ao_span(galileo_span: GalileoSpan) -> Generator[trace.Span, Any _TRACE_PROVIDER_CONTEXT_VAR.set(cast(TracerProvider, tracer_provider)) tracer = tracer_provider.get_tracer("galileo-tracer") with tracer.start_as_current_span(galileo_span.name) as span: - yield span - # Set dataset attributes for ground truth/reference output support - _apply_dataset_attributes( - span, galileo_span.dataset_input, galileo_span.dataset_output, galileo_span.dataset_metadata - ) - if isinstance(galileo_span, RetrieverSpan): - _set_retriever_span_attributes(span, galileo_span) - elif isinstance(galileo_span, ToolSpan): - _set_tool_span_attributes(span, galileo_span) - elif isinstance(galileo_span, WorkflowSpan): - _set_workflow_span_attributes(span, galileo_span) + try: + yield span + finally: + for key, value in build_span_attributes(galileo_span, _session_id_context.get(None)).items(): + span.set_attribute(key, value) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py new file mode 100644 index 00000000..389cae20 --- /dev/null +++ b/tests/test_attribute_mapping.py @@ -0,0 +1,222 @@ +import json +from uuid import uuid4 + +import pytest + +from galileo_core.schemas.logging.span import ( + AgentSpan, + AgentType, + LlmMetrics, + LlmSpan, + RetrieverSpan, + ToolSpan, + WorkflowSpan, +) +from galileo_core.schemas.shared.document import Document +from splunk_ao.converter.attribute_mapping import ( + CONTENT_ALIAS_BY_GEN_AI, + SPLUNK_ALIAS_BY_GEN_AI, + build_span_attributes, + normalize_attributes_for_export, +) +from splunk_ao.logger.control import ControlResult, ControlSpan + + +def test_llm_mapping_covers_content_request_response_usage_and_units() -> None: + span = LlmSpan( + input="prompt", + output="answer", + model="gpt-5-nano", + temperature=0.0, + finish_reason="stop", + tools=[{"type": "function", "name": "search"}], + metrics=LlmMetrics( + num_input_tokens=12, + num_output_tokens=8, + num_total_tokens=20, + time_to_first_token_ns=250_000_000, + input_cost=0.01, + output_cost=0.02, + total_cost=0.03, + cache_read_input_tokens=4, + ), + ) + + attrs = build_span_attributes(span) + + assert attrs["gen_ai.operation.name"] == "chat" + assert json.loads(attrs["gen_ai.input.messages"])[0]["content"] == "prompt" + assert json.loads(attrs["gen_ai.output.messages"])["content"] == "answer" + assert attrs["gen_ai.request.model"] == "gpt-5-nano" + assert attrs["gen_ai.request.temperature"] == 0.0 + assert attrs["gen_ai.response.finish_reasons"] == ("stop",) + assert attrs["gen_ai.usage.input_tokens"] == 12 + assert attrs["gen_ai.usage.output_tokens"] == 8 + assert attrs["splunk_ao.llm.usage.total_tokens"] == 20 + assert attrs["gen_ai.response.time_to_first_chunk"] == 0.25 + assert attrs["splunk_ao.llm.time_to_first_token_ns"] == 250_000_000 + assert attrs["splunk_ao.llm.cost.total_usd"] == 0.03 + assert attrs["gen_ai.usage.cache_read.input_tokens"] == 4 + assert json.loads(attrs["gen_ai.tool.definitions"])[0]["name"] == "search" + + +def test_common_mapping_uses_canonical_keys_and_omits_external_id() -> None: + session_id = uuid4() + span = ToolSpan( + name="search", + input='{"q":"x"}', + output="result", + status_code=500, + user_metadata={"team": "checkout"}, + tags=["production"], + external_id="not-an-otel-attribute", + dataset_input="question", + dataset_output="expected", + dataset_metadata={"split": "test"}, + session_id=session_id, + step_number=0, + ) + + attrs = build_span_attributes(span) + + assert attrs["gen_ai.operation.name"] == "execute_tool" + assert attrs["gen_ai.tool.name"] == "search" + assert attrs["gen_ai.tool.call.arguments"] == '{"q":"x"}' + assert attrs["gen_ai.tool.call.result"] == "result" + assert attrs["gen_ai.conversation.id"] == str(session_id) + assert json.loads(attrs["splunk_ao.metadata"]) == {"team": "checkout"} + assert attrs["splunk_ao.tags"] == ("production",) + assert attrs["splunk_ao.status_code"] == 500 + assert attrs["error.type"] == "500" + assert attrs["splunk_ao.step_number"] == 0 + assert attrs["splunk_ao.dataset.input"] == "question" + assert attrs["splunk_ao.dataset.output"] == "expected" + assert json.loads(attrs["splunk_ao.dataset.metadata"]) == {"split": "test"} + assert not any("external_id" in key for key in attrs) + + +def test_explicit_session_context_precedes_span_session() -> None: + span = ToolSpan(name="search", session_id=uuid4()) + + assert build_span_attributes(span, session_id="context-session")["gen_ai.conversation.id"] == "context-session" + + +def test_retriever_mapping_uses_query_and_documents() -> None: + span = RetrieverSpan( + name="vector-search", input="what is RAG?", output=[Document(content="doc", metadata={"source": "kb"})] + ) + + attrs = build_span_attributes(span) + + assert attrs["gen_ai.operation.name"] == "retrieval" + assert attrs["gen_ai.retrieval.query.text"] == "what is RAG?" + assert json.loads(attrs["gen_ai.retrieval.documents"])[0]["content"] == "doc" + assert attrs["splunk_ao.retrieval.documents.count"] == 1 + assert attrs["db.operation"] == "search" + assert "gen_ai.output.messages" not in attrs + + +@pytest.mark.parametrize( + ("span", "operation_key", "operation", "name_key", "name"), + [ + ( + WorkflowSpan(name="workflow", input="question", output="answer"), + "gen_ai.operation.name", + "invoke_workflow", + "gen_ai.workflow.name", + "workflow", + ), + ( + AgentSpan(name="planner", input="question", output="answer", agent_type=AgentType.planner), + "gen_ai.operation.name", + "invoke_agent", + "gen_ai.agent.name", + "planner", + ), + ], +) +def test_orchestration_mapping(span, operation_key: str, operation: str, name_key: str, name: str) -> None: + attrs = build_span_attributes(span) + + assert attrs[operation_key] == operation + assert attrs[name_key] == name + assert attrs["gen_ai.input.messages"] == "question" + assert attrs["gen_ai.output.messages"] == "answer" + + +def test_control_mapping_preserves_structured_content() -> None: + span = ControlSpan( + name="guardrail", + input={"text": "question"}, + output=ControlResult(action="observe", matched=True, confidence=0.9), + ) + + attrs = build_span_attributes(span) + + assert attrs["splunk_ao.operation.name"] == "control" + assert json.loads(attrs["gen_ai.input.messages"]) == {"text": "question"} + assert json.loads(attrs["gen_ai.output.messages"])["matched"] is True + + +def test_normalizer_duplicates_ordinary_attributes_and_relocates_all_content() -> None: + source = { + "gen_ai.request.model": "gpt-4o", + "gen_ai.usage.input_tokens": 42, + **{key: f"value-{index}" for index, key in enumerate(CONTENT_ALIAS_BY_GEN_AI)}, + } + + result = normalize_attributes_for_export(source) + + assert result["gen_ai.request.model"] == result["splunk_ao.request.model"] == "gpt-4o" + assert result["gen_ai.usage.input_tokens"] == result["splunk_ao.llm.usage.input_tokens"] == 42 + for source_key, destination_key in CONTENT_ALIAS_BY_GEN_AI.items(): + assert source_key not in result + assert result[destination_key] == source[source_key] + + +def test_normalizer_gen_ai_wins_collisions_and_sdk_marker_is_authoritative() -> None: + result = normalize_attributes_for_export( + { + "gen_ai.request.model": "canonical", + "splunk_ao.request.model": "stale", + "gen_ai.system": "legacy-provider", + "gen_ai.provider.name": "current-provider", + "splunk_ao.provider.name": "stale-provider", + "splunk_ao.system": "other-sdk", + } + ) + + assert result["splunk_ao.request.model"] == "canonical" + assert result["splunk_ao.provider.name"] == "current-provider" + assert result["splunk_ao.system"] == "splunk_ao_python" + + +def test_normalizer_converts_first_chunk_seconds_to_splunk_nanoseconds() -> None: + result = normalize_attributes_for_export({"gen_ai.response.time_to_first_chunk": 0.125}) + + assert result["gen_ai.response.time_to_first_chunk"] == 0.125 + assert result["splunk_ao.llm.time_to_first_token_ns"] == 125_000_000 + + +def test_normalizer_is_idempotent() -> None: + source = {"gen_ai.operation.name": "chat", "gen_ai.input.messages": "input-json", "custom.attribute": "unchanged"} + + once = normalize_attributes_for_export(source) + + assert normalize_attributes_for_export(once) == once + + +def test_normalizer_can_be_disabled_for_developer_comparison() -> None: + source = { + "gen_ai.request.model": "gpt-4o", + "gen_ai.input.messages": "input-json", + "splunk_ao.system": "source-value", + } + + assert normalize_attributes_for_export(source, enabled=False) == source + + +def test_every_alias_uses_an_explicit_destination_namespace() -> None: + assert SPLUNK_ALIAS_BY_GEN_AI + assert all(source.startswith("gen_ai.") for source in SPLUNK_ALIAS_BY_GEN_AI) + assert all(destination.startswith("splunk_ao.") for destination in SPLUNK_ALIAS_BY_GEN_AI.values()) diff --git a/tests/test_exporter_config.py b/tests/test_exporter_config.py index 12049145..7b8dc705 100644 --- a/tests/test_exporter_config.py +++ b/tests/test_exporter_config.py @@ -5,6 +5,7 @@ from splunk_ao.deployment import StandaloneConfig from splunk_ao.exporter.config import RoutingAttrs, routing_resource_attributes +from splunk_ao.exporter.span_transform import NormalizingSpanExporter from splunk_ao.exporter.standalone import build_standalone_exporter, resolve_standalone_exporter_config from splunk_ao.logger import logger as logger_module from splunk_ao.logger.logger import SplunkAOLogger @@ -132,7 +133,8 @@ def exporter_factory(**kwargs: Any) -> object: make_standalone_cfg(), make_routing(project_name="p"), _exporter_factory=exporter_factory ) - assert exporter is expected_exporter + assert isinstance(exporter, NormalizingSpanExporter) + assert exporter.delegate is expected_exporter assert captured == { "endpoint": "https://api.demo.galileocloud.io/otel/v1/traces", "headers": {"Splunk-AO-API-Key": "key", "project": "p"}, diff --git a/tests/test_exporter_o11y.py b/tests/test_exporter_o11y.py index f34c6802..47672ee0 100644 --- a/tests/test_exporter_o11y.py +++ b/tests/test_exporter_o11y.py @@ -7,6 +7,7 @@ from splunk_ao.deployment import O11yConfig from splunk_ao.exporter.config import RoutingAttrs from splunk_ao.exporter.o11y import build_o11y_exporter, resolve_o11y_exporter_config +from splunk_ao.exporter.span_transform import NormalizingSpanExporter from splunk_ao.shared.exceptions import MissingConfigurationError @@ -98,7 +99,8 @@ def exporter_factory(**kwargs: Any) -> object: _exporter_factory=exporter_factory, ) - assert exporter is expected_exporter + assert isinstance(exporter, NormalizingSpanExporter) + assert exporter.delegate is expected_exporter assert captured == { "endpoint": "https://ingest.us1.observability.splunkcloud.com/v2/trace/otlp", "headers": {"X-SF-Token": "tok", "projectid": "pid", "logstreamid": "lsid"}, diff --git a/tests/test_otel.py b/tests/test_otel.py index 5b114048..69826847 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -4,8 +4,9 @@ import pytest from galileo_core.schemas.logging.llm import Message, MessageRole -from galileo_core.schemas.logging.span import ToolSpan, WorkflowSpan +from galileo_core.schemas.logging.span import AgentSpan, LlmSpan, RetrieverSpan, ToolSpan, WorkflowSpan from galileo_core.schemas.shared.document import Document +from splunk_ao.converter import build_span_attributes from splunk_ao.decorator import ( _dataset_input_context, _dataset_metadata_context, @@ -21,8 +22,6 @@ _TRACE_PROVIDER_CONTEXT_VAR, SplunkAOOTLPExporter, SplunkAOSpanProcessor, - _set_tool_span_attributes, - _set_workflow_span_attributes, start_splunk_ao_span, ) @@ -256,13 +255,13 @@ def test_processor_on_start_sets_content_not_routing_attributes(self, mock_proce assert mock_span.set_attribute.call_count == 1 actual_calls = {(args[0], args[1]) for args, _ in mock_span.set_attribute.call_args_list} - assert ("splunk_ao.session.id", "test-session") in actual_calls + assert ("gen_ai.conversation.id", "test-session") in actual_calls routing_keys = {"splunk_ao.project.name", "splunk_ao.logstream.name", "splunk_ao.experiment.id"} assert not routing_keys.intersection(key for key, _ in actual_calls) class TestSetToolSpanAttributes: - """Test suite for _set_tool_span_attributes function.""" + """Test the canonical tool-span builder.""" def test_tool_span_with_all_fields(self): """Test setting attributes when all ToolSpan fields are populated.""" @@ -274,41 +273,27 @@ def test_tool_span_with_all_fields(self): tool_call_id="call-123", status_code=200, ) - mock_otel_span = Mock() + attrs = build_span_attributes(tool_span) - # When: setting tool span attributes - _set_tool_span_attributes(mock_otel_span, tool_span) - - # Then: all attributes are set correctly - calls = {args[0]: args[1] for args, _ in mock_otel_span.set_attribute.call_args_list} - assert calls["gen_ai.operation.name"] == "execute_tool" - assert calls["gen_ai.tool.name"] == "test-tool" - assert calls["gen_ai.tool.call.arguments"] == "tool input data" - assert calls["gen_ai.tool.call.result"] == "tool output result" - assert calls["gen_ai.input.messages"] == json.dumps([{"role": "tool", "content": "tool input data"}]) - assert calls["gen_ai.output.messages"] == json.dumps([{"role": "tool", "content": "tool output result"}]) - assert calls["gen_ai.tool.call.id"] == "call-123" - assert mock_otel_span.set_attribute.call_count == 7 + assert attrs["gen_ai.operation.name"] == "execute_tool" + assert attrs["gen_ai.tool.name"] == "test-tool" + assert attrs["gen_ai.tool.call.arguments"] == "tool input data" + assert attrs["gen_ai.tool.call.result"] == "tool output result" + assert attrs["gen_ai.tool.call.id"] == "call-123" + assert "gen_ai.input.messages" not in attrs + assert "gen_ai.output.messages" not in attrs def test_tool_span_with_only_input(self): """Test setting attributes when only input is provided.""" # Given: a ToolSpan with only input (output and tool_call_id are None) tool_span = ToolSpan(name="test-tool", input="tool input only", output=None, tool_call_id=None, status_code=200) - mock_otel_span = Mock() + attrs = build_span_attributes(tool_span) - # When: setting tool span attributes - _set_tool_span_attributes(mock_otel_span, tool_span) - - # Then: operation name, tool name, and input attributes are set, but not output or tool_call_id - calls = {args[0]: args[1] for args, _ in mock_otel_span.set_attribute.call_args_list} - assert calls["gen_ai.operation.name"] == "execute_tool" - assert calls["gen_ai.tool.name"] == "test-tool" - assert calls["gen_ai.tool.call.arguments"] == "tool input only" - assert calls["gen_ai.input.messages"] == json.dumps([{"role": "tool", "content": "tool input only"}]) - assert "gen_ai.tool.call.result" not in calls - assert "gen_ai.output.messages" not in calls - assert "gen_ai.tool.call.id" not in calls - assert mock_otel_span.set_attribute.call_count == 4 + assert attrs["gen_ai.operation.name"] == "execute_tool" + assert attrs["gen_ai.tool.name"] == "test-tool" + assert attrs["gen_ai.tool.call.arguments"] == "tool input only" + assert "gen_ai.tool.call.result" not in attrs + assert "gen_ai.tool.call.id" not in attrs def test_tool_span_with_output_no_tool_call_id(self): """Test setting attributes when output is provided but tool_call_id is None.""" @@ -316,21 +301,13 @@ def test_tool_span_with_output_no_tool_call_id(self): tool_span = ToolSpan( name="test-tool", input="tool input", output="tool output", tool_call_id=None, status_code=200 ) - mock_otel_span = Mock() + attrs = build_span_attributes(tool_span) - # When: setting tool span attributes - _set_tool_span_attributes(mock_otel_span, tool_span) - - # Then: operation name, tool name, input, and output attributes are set, but not tool_call_id - calls = {args[0]: args[1] for args, _ in mock_otel_span.set_attribute.call_args_list} - assert calls["gen_ai.operation.name"] == "execute_tool" - assert calls["gen_ai.tool.name"] == "test-tool" - assert calls["gen_ai.tool.call.arguments"] == "tool input" - assert calls["gen_ai.tool.call.result"] == "tool output" - assert calls["gen_ai.input.messages"] == json.dumps([{"role": "tool", "content": "tool input"}]) - assert calls["gen_ai.output.messages"] == json.dumps([{"role": "tool", "content": "tool output"}]) - assert "gen_ai.tool.call.id" not in calls - assert mock_otel_span.set_attribute.call_count == 6 + assert attrs["gen_ai.operation.name"] == "execute_tool" + assert attrs["gen_ai.tool.name"] == "test-tool" + assert attrs["gen_ai.tool.call.arguments"] == "tool input" + assert attrs["gen_ai.tool.call.result"] == "tool output" + assert "gen_ai.tool.call.id" not in attrs class TestStartGalileoSpan: @@ -372,8 +349,6 @@ def test_start_splunk_ao_span_dispatches_tool_span(self): assert calls["gen_ai.tool.name"] == "my-tool" assert calls["gen_ai.tool.call.arguments"] == "tool input data" assert calls["gen_ai.tool.call.result"] == "tool output result" - assert calls["gen_ai.input.messages"] == json.dumps([{"role": "tool", "content": "tool input data"}]) - assert calls["gen_ai.output.messages"] == json.dumps([{"role": "tool", "content": "tool output result"}]) assert calls["gen_ai.tool.call.id"] == "call-789" def test_start_splunk_ao_span_tool_span_with_none_output(self): @@ -398,65 +373,82 @@ def test_start_splunk_ao_span_tool_span_with_none_output(self): assert calls["gen_ai.operation.name"] == "execute_tool" assert calls["gen_ai.tool.name"] == "minimal-tool" assert calls["gen_ai.tool.call.arguments"] == "just input" - assert calls["gen_ai.input.messages"] == json.dumps([{"role": "tool", "content": "just input"}]) assert "gen_ai.tool.call.result" not in calls - assert "gen_ai.output.messages" not in calls assert "gen_ai.tool.call.id" not in calls + @pytest.mark.parametrize( + "galileo_span", + [ + LlmSpan(input="prompt", output="answer", model="gpt-4o"), + ToolSpan(name="search", input="query", output="result"), + RetrieverSpan(name="retrieval", input="query", output=[Document(content="result", metadata={})]), + WorkflowSpan(name="workflow", input="question", output="answer"), + AgentSpan(name="agent", input="question", output="answer"), + ], + ) + def test_start_span_applies_canonical_builder_for_every_supported_type(self, galileo_span): + expected = build_span_attributes(galileo_span, session_id="session-id") + mock_otel_span = Mock() + mock_tracer = Mock() + mock_tracer.start_as_current_span.return_value.__enter__ = Mock(return_value=mock_otel_span) + mock_tracer.start_as_current_span.return_value.__exit__ = Mock(return_value=False) + mock_provider = Mock() + mock_provider.get_tracer.return_value = mock_tracer + _TRACE_PROVIDER_CONTEXT_VAR.set(mock_provider) + token = _session_id_context.set("session-id") + + try: + with start_splunk_ao_span(galileo_span): + pass + finally: + _session_id_context.reset(token) + + calls = {args[0]: args[1] for args, _ in mock_otel_span.set_attribute.call_args_list} + assert {key: calls[key] for key in expected} == expected + + def test_start_span_applies_canonical_attributes_when_body_raises(self): + galileo_span = ToolSpan(name="search", input="query", output="result") + mock_otel_span = Mock() + mock_tracer = Mock() + mock_tracer.start_as_current_span.return_value.__enter__ = Mock(return_value=mock_otel_span) + mock_tracer.start_as_current_span.return_value.__exit__ = Mock(return_value=False) + mock_provider = Mock() + mock_provider.get_tracer.return_value = mock_tracer + _TRACE_PROVIDER_CONTEXT_VAR.set(mock_provider) + + with pytest.raises(RuntimeError, match="failure"), start_splunk_ao_span(galileo_span): + raise RuntimeError("failure") + + calls = {args[0]: args[1] for args, _ in mock_otel_span.set_attribute.call_args_list} + assert calls["gen_ai.operation.name"] == "execute_tool" + assert calls["gen_ai.tool.name"] == "search" + class TestWorkflowSpanAttributes: """Test suite for WorkflowSpan OpenTelemetry attribute mapping.""" - @pytest.fixture - def mock_dependencies(self): - """Set up mocks for testing workflow span attributes.""" - with patch("splunk_ao.otel.trace") as mock_trace_module, patch("splunk_ao.otel.json") as mock_json_module: - mock_span = Mock() - mock_json_module.dumps.return_value = '"test"' - yield {"span": mock_span, "trace": mock_trace_module, "json": mock_json_module} - - def test_workflow_span_with_string_input_output(self, mock_dependencies): + def test_workflow_span_with_string_input_output(self): """Test WorkflowSpan with string input and output.""" # Given: a WorkflowSpan with string input and output workflow_span = WorkflowSpan(name="test-workflow", input="input text", output="output text", status_code=200) - mock_span = mock_dependencies["span"] - mock_json = mock_dependencies["json"] - - # When: setting workflow span attributes - _set_workflow_span_attributes(mock_span, workflow_span) - - # Then: input and output should be wrapped in message format - assert mock_span.set_attribute.call_count == 2 - - # Check first call (input) - input_call = mock_span.set_attribute.call_args_list[0] - assert input_call[0][0] == "gen_ai.input.messages" - mock_json.dumps.assert_any_call([{"role": "user", "content": "input text"}]) + attrs = build_span_attributes(workflow_span) - # Check second call (output) - output_call = mock_span.set_attribute.call_args_list[1] - assert output_call[0][0] == "gen_ai.output.messages" - mock_json.dumps.assert_any_call([{"role": "assistant", "content": "output text"}]) + assert attrs["gen_ai.operation.name"] == "invoke_workflow" + assert attrs["gen_ai.workflow.name"] == "test-workflow" + assert attrs["gen_ai.input.messages"] == "input text" + assert attrs["gen_ai.output.messages"] == "output text" - def test_workflow_span_with_message_input_output(self, mock_dependencies): + def test_workflow_span_with_message_input_output(self): """Test WorkflowSpan with Message input and output.""" # Given: a WorkflowSpan with Message input and output input_msg = Message(role=MessageRole.user, content="user question") workflow_span = WorkflowSpan(name="test-workflow", input=[input_msg], output=input_msg, status_code=200) - mock_span = mock_dependencies["span"] - mock_dependencies["json"] + attrs = build_span_attributes(workflow_span) - # When: setting workflow span attributes - _set_workflow_span_attributes(mock_span, workflow_span) + assert json.loads(attrs["gen_ai.input.messages"])[0]["content"] == "user question" + assert json.loads(attrs["gen_ai.output.messages"])["content"] == "user question" - # Then: input and output should serialize Message objects - assert mock_span.set_attribute.call_count == 2 - input_call = mock_span.set_attribute.call_args_list[0] - assert input_call[0][0] == "gen_ai.input.messages" - output_call = mock_span.set_attribute.call_args_list[1] - assert output_call[0][0] == "gen_ai.output.messages" - - def test_workflow_span_with_document_sequence_output(self, mock_dependencies): + def test_workflow_span_with_document_sequence_output(self): """Test WorkflowSpan with Document sequence output.""" # Given: a WorkflowSpan with string input and Document sequence output documents = [ @@ -469,37 +461,27 @@ def test_workflow_span_with_document_sequence_output(self, mock_dependencies): workflow_span = WorkflowSpan.model_construct( name="test-workflow", input="query", output=documents, status_code=200 ) - mock_span = mock_dependencies["span"] - mock_dependencies["json"] - - # When: setting workflow span attributes - _set_workflow_span_attributes(mock_span, workflow_span) + attrs = build_span_attributes(workflow_span) - # Then: output should be wrapped in assistant message with documents - assert mock_span.set_attribute.call_count == 2 - output_call = mock_span.set_attribute.call_args_list[1] - assert output_call[0][0] == "gen_ai.output.messages" + assert [document["content"] for document in json.loads(attrs["gen_ai.output.messages"])] == [ + "doc1 content", + "doc2 content", + ] - def test_workflow_span_with_none_output(self, mock_dependencies): + def test_workflow_span_with_none_output(self): """Test WorkflowSpan with None output (should not set output attribute).""" # Given: a WorkflowSpan with None output workflow_span = WorkflowSpan(name="test-workflow", input="input text", output=None, status_code=200) - mock_span = mock_dependencies["span"] + attrs = build_span_attributes(workflow_span) - # When: setting workflow span attributes - _set_workflow_span_attributes(mock_span, workflow_span) + assert attrs["gen_ai.input.messages"] == "input text" + assert "gen_ai.output.messages" not in attrs - # Then: only input attribute should be set, not output - assert mock_span.set_attribute.call_count == 1 - input_call = mock_span.set_attribute.call_args_list[0] - assert input_call[0][0] == "gen_ai.input.messages" - - def test_workflow_span_in_start_splunk_ao_span(self, mock_dependencies): + def test_workflow_span_in_start_splunk_ao_span(self): """Test that WorkflowSpan is handled in start_splunk_ao_span context manager.""" # Given: a WorkflowSpan workflow_span = WorkflowSpan(name="test-workflow", input="input", output="output", status_code=200) - mock_span = mock_dependencies["span"] - mock_dependencies["json"] + mock_span = Mock() # Setup the mock tracer mock_tracer = Mock() diff --git a/tests/test_otel_native_paths.py b/tests/test_otel_native_paths.py index 34daa22b..2fb4b2d0 100644 --- a/tests/test_otel_native_paths.py +++ b/tests/test_otel_native_paths.py @@ -288,8 +288,11 @@ def test_exporter_preserves_every_unaffected_span_field() -> None: assert getattr(exported, field) == getattr(source, field) assert exported.resource.schema_url == source.resource.schema_url assert exported.attributes["gen_ai.request.model"] == "gpt-4o" + assert exported.attributes["splunk_ao.request.model"] == "gpt-4o" assert exported.attributes["gen_ai.provider.name"] == "openai" + assert exported.attributes["splunk_ao.provider.name"] == "openai" assert exported.attributes["gen_ai.system"] == "legacy-upstream-provider" + assert exported.attributes["splunk_ao.system"] == "splunk_ao_python" assert exported.attributes["custom.attribute"] == "preserved" exporter.shutdown() @@ -321,7 +324,7 @@ def test_processor_does_not_put_routing_on_span_attributes() -> None: calls = {args[0]: args[1] for args, _ in span.set_attribute.call_args_list} assert not ROUTING_KEYS.intersection(calls) - assert calls["splunk_ao.session.id"] == "session-id" + assert calls["gen_ai.conversation.id"] == "session-id" assert calls["splunk_ao.dataset.input"] == "question" processor.shutdown() diff --git a/tests/test_span_transform.py b/tests/test_span_transform.py new file mode 100644 index 00000000..bda12811 --- /dev/null +++ b/tests/test_span_transform.py @@ -0,0 +1,153 @@ +from collections.abc import Sequence + +import pytest +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from opentelemetry.trace import SpanContext, SpanKind, TraceFlags +from opentelemetry.trace.status import Status, StatusCode + +from splunk_ao.exporter.span_transform import NormalizingSpanExporter, copy_span_for_export + + +class RecordingExporter(SpanExporter): + def __init__(self) -> None: + self.batches: list[Sequence[ReadableSpan]] = [] + self.flushes: list[int] = [] + self.shutdowns = 0 + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + self.batches.append(spans) + return SpanExportResult.SUCCESS + + def force_flush(self, timeout_millis: int = 30000) -> bool: + self.flushes.append(timeout_millis) + return True + + def shutdown(self) -> None: + self.shutdowns += 1 + + +def make_span(attributes: dict[str, object] | None = None) -> ReadableSpan: + return ReadableSpan( + name="span", + context=SpanContext( + trace_id=0x1234567890ABCDEF1234567890ABCDEF, + span_id=0x1234567890ABCDEF, + is_remote=False, + trace_flags=TraceFlags.SAMPLED, + ), + resource=Resource( + {"service.name": "checkout", "splunk_ao.project.name": "stale"}, + schema_url="https://opentelemetry.io/schemas/1.38.0", + ), + attributes=attributes or {"gen_ai.request.model": "gpt-4o"}, + kind=SpanKind.CLIENT, + status=Status(StatusCode.OK), + start_time=1, + end_time=2, + instrumentation_scope=InstrumentationScope("instrumentation", "1.0"), + ) + + +def test_copy_span_for_export_is_immutable_and_combines_normalization_with_routing() -> None: + source = make_span( + { + "gen_ai.request.model": "gpt-4o", + "gen_ai.input.messages": "input-json", + "splunk_ao.project.name": "stale-span-routing", + } + ) + source_attributes = dict(source.attributes or {}) + source_resource = source.resource + + exported = copy_span_for_export( + source, Resource({"splunk_ao.project.id": "project-id", "splunk_ao.logstream.id": "log-stream-id"}) + ) + + assert exported is not source + assert dict(source.attributes or {}) == source_attributes + assert source.resource is source_resource + assert exported.attributes["splunk_ao.request.model"] == "gpt-4o" + assert exported.attributes["splunk_ao.input.messages"] == "input-json" + assert "gen_ai.input.messages" not in exported.attributes + assert "splunk_ao.project.name" not in exported.attributes + assert exported.resource.attributes["splunk_ao.project.id"] == "project-id" + assert exported.resource.attributes["splunk_ao.logstream.id"] == "log-stream-id" + assert "splunk_ao.project.name" not in exported.resource.attributes + assert exported.resource.attributes["service.name"] == "checkout" + + +def test_copy_span_preserves_unaffected_readable_span_fields() -> None: + source = make_span() + exported = copy_span_for_export(source) + + for field in ( + "name", + "context", + "parent", + "events", + "links", + "kind", + "status", + "start_time", + "end_time", + "instrumentation_info", + "instrumentation_scope", + ): + assert getattr(exported, field) == getattr(source, field) + assert exported.resource.schema_url == source.resource.schema_url + + +def test_exporter_normalizes_once_and_delegates_lifecycle_once() -> None: + delegate = RecordingExporter() + exporter = NormalizingSpanExporter(delegate, Resource({"splunk_ao.project.name": "project"})) + + assert exporter.export((make_span(),)) == SpanExportResult.SUCCESS + assert exporter.force_flush(1234) is True + exporter.shutdown() + + assert len(delegate.batches) == 1 + assert delegate.batches[0][0].attributes["splunk_ao.request.model"] == "gpt-4o" + assert delegate.batches[0][0].resource.attributes["splunk_ao.project.name"] == "project" + assert delegate.flushes == [1234] + assert delegate.shutdowns == 1 + + +@pytest.mark.parametrize("value", ["0", "false", "FALSE", "no", "off"]) +def test_private_environment_switch_disables_only_attribute_normalization( + monkeypatch: pytest.MonkeyPatch, value: str +) -> None: + monkeypatch.setenv("SPLUNK_AO_DEV_ENABLE_ATTRIBUTE_NORMALIZATION", value) + delegate = RecordingExporter() + exporter = NormalizingSpanExporter(delegate, Resource({"splunk_ao.project.name": "project"})) + source = make_span({"gen_ai.input.messages": "input-json"}) + + exporter.export((source,)) + exported = delegate.batches[0][0] + + assert exported.attributes == source.attributes + assert "splunk_ao.system" not in exported.attributes + assert exported.resource.attributes["splunk_ao.project.name"] == "project" + + +def test_private_environment_switch_is_read_at_exporter_construction(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SPLUNK_AO_DEV_ENABLE_ATTRIBUTE_NORMALIZATION", "false") + delegate = RecordingExporter() + exporter = NormalizingSpanExporter(delegate) + monkeypatch.setenv("SPLUNK_AO_DEV_ENABLE_ATTRIBUTE_NORMALIZATION", "true") + + exporter.export((make_span(),)) + + assert "splunk_ao.request.model" not in delegate.batches[0][0].attributes + + +def test_default_normalization_is_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("SPLUNK_AO_DEV_ENABLE_ATTRIBUTE_NORMALIZATION", raising=False) + delegate = RecordingExporter() + exporter = NormalizingSpanExporter(delegate) + + exporter.export((make_span(),)) + + assert delegate.batches[0][0].attributes["splunk_ao.system"] == "splunk_ao_python" From f1420486114472722bd6cf428d5321b05d15b5d4 Mon Sep 17 00:00:00 2001 From: pradystar Date: Fri, 24 Jul 2026 13:26:06 -0700 Subject: [PATCH 03/10] fix(otel): correct structured GenAI attribute serialization --- src/splunk_ao/converter/attribute_mapping.py | 150 ++++++++++++++++-- tests/test_attribute_mapping.py | 153 +++++++++++++++++-- tests/test_otel.py | 43 +++--- 3 files changed, 306 insertions(+), 40 deletions(-) diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 74d60ceb..3381efe6 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -93,6 +93,138 @@ def _content_value(value: Any) -> str: return value if isinstance(value, str) else _json_string(value) +def _mapping_value(value: Any) -> dict[str, Any] | None: + if isinstance(value, BaseModel): + return value.model_dump(mode="json", exclude_none=True) + if isinstance(value, Mapping): + return {str(key): _json_compatible(item) for key, item in value.items()} + return None + + +def _parse_json_value(value: Any) -> Any: + if not isinstance(value, str): + return _json_compatible(value) + try: + return json.loads(value) + except (TypeError, ValueError): + return value + + +def _text_part(value: Any) -> dict[str, Any]: + content = value if isinstance(value, str) else _json_string(value) + return {"type": "text", "content": content} + + +def _content_part(value: Any) -> dict[str, Any] | None: + part = _mapping_value(value) + if part is None or "type" not in part: + return None + + part_type = _json_compatible(part["type"]) + part["type"] = str(part_type) + if part_type == "text" and "content" not in part and "text" in part: + part["content"] = part.pop("text") + return part + + +def _content_parts(value: Any) -> list[dict[str, Any]]: + part = _content_part(value) + if part is not None: + return [part] + + if isinstance(value, Sequence) and not isinstance(value, str | bytes | bytearray): + parts = [_content_part(item) for item in value] + if all(item is not None for item in parts): + return [item for item in parts if item is not None] + + return [_text_part(value)] + + +def _tool_call_part(value: Any) -> dict[str, Any]: + call = _mapping_value(value) or {} + function = _mapping_value(call.pop("function", None)) or {} + result = {**call, "type": "tool_call"} + if "name" in function: + result["name"] = function.pop("name") + if "arguments" in function: + result["arguments"] = _parse_json_value(function.pop("arguments")) + result.update(function) + return result + + +def _mapped_message(source: dict[str, Any], default_role: str) -> dict[str, Any]: + role = _json_compatible(source.pop("role")) + content = source.pop("content", "") + tool_call_id = source.pop("tool_call_id", None) + tool_calls = source.pop("tool_calls", None) + + if role == "tool": + response = {"type": "tool_call_response", "response": _parse_json_value(content)} + if tool_call_id is not None: + response["id"] = str(tool_call_id) + parts = [response] + else: + parts = [] if content == "" and tool_calls else _content_parts(content) + + if tool_calls: + parts.extend(_tool_call_part(tool_call) for tool_call in tool_calls) + + return {**source, "role": str(role), "parts": parts} + + +def _message(value: Any, default_role: str) -> dict[str, Any]: + source = _mapping_value(value) + if source is None or "role" not in source: + return {"role": default_role, "parts": _content_parts(value)} + return _mapped_message(source, default_role) + + +def _message_sequence(value: Any, default_role: str) -> list[dict[str, Any]]: + if not isinstance(value, Sequence) or isinstance(value, str | bytes | bytearray): + return [_message(value, default_role)] + if not value: + return [] + + messages = [_mapping_value(item) for item in value] + if all(message is not None and "role" in message for message in messages): + return [_mapped_message(message, default_role) for message in messages if message is not None] + return [_message(value, default_role)] + + +def _input_messages(value: Any) -> str: + return _json_string(_message_sequence(value, "user")) + + +def _output_messages(value: Any, finish_reason: str | None = None) -> str: + messages = _message_sequence(value, "assistant") + for message in messages: + source_finish_reason = message.get("finish_reason") + message["finish_reason"] = finish_reason or source_finish_reason or "unknown" + return _json_string(messages) + + +def _tool_definitions(value: Sequence[Any]) -> str: + definitions: list[Any] = [] + for tool in value: + definition = _mapping_value(tool) + if definition is None: + definitions.append(_json_compatible(tool)) + continue + + function = _mapping_value(definition.pop("function", None)) + if function is not None: + definition = {**function, **definition} + definition.setdefault("type", "function") + definitions.append(definition) + return _json_string(definitions) + + +def _object_content(value: Any) -> str: + parsed = _parse_json_value(value) + content = parsed if isinstance(parsed, Mapping) else {"value": parsed} + return _json_string(content) + + def _set_if_present(attrs: MutableMapping[str, AttributeValue], key: str, value: AttributeValue | None) -> None: if value is not None: attrs[key] = value @@ -149,8 +281,8 @@ def _set_operation(attrs: MutableMapping[str, AttributeValue], span_type: StepTy def set_llm_attributes(attrs: MutableMapping[str, AttributeValue], span: LlmSpan) -> None: """Map LLM request, response, content, and metric fields.""" _set_operation(attrs, StepType.llm) - attrs["gen_ai.input.messages"] = _content_value(span.input) - attrs["gen_ai.output.messages"] = _content_value(span.output) + attrs["gen_ai.input.messages"] = _input_messages(span.input) + attrs["gen_ai.output.messages"] = _output_messages(span.output, span.finish_reason) _set_if_present(attrs, "gen_ai.request.model", span.model) _set_if_present(attrs, "gen_ai.request.temperature", span.temperature) @@ -171,7 +303,7 @@ def set_llm_attributes(attrs: MutableMapping[str, AttributeValue], span: LlmSpan if span.finish_reason is not None: attrs["gen_ai.response.finish_reasons"] = (span.finish_reason,) if span.tools is not None: - attrs["gen_ai.tool.definitions"] = _json_string(span.tools) + attrs["gen_ai.tool.definitions"] = _tool_definitions(span.tools) if span.events is not None: attrs["splunk_ao.llm.events"] = _json_string(span.events) @@ -208,9 +340,9 @@ def set_tool_attributes(attrs: MutableMapping[str, AttributeValue], span: ToolSp """Map tool execution fields.""" _set_operation(attrs, StepType.tool) attrs["gen_ai.tool.name"] = span.name - attrs["gen_ai.tool.call.arguments"] = _content_value(span.input) + attrs["gen_ai.tool.call.arguments"] = _object_content(span.input) if span.output is not None: - attrs["gen_ai.tool.call.result"] = _content_value(span.output) + attrs["gen_ai.tool.call.result"] = _object_content(span.output) _set_if_present(attrs, "gen_ai.tool.call.id", span.tool_call_id) @@ -226,9 +358,9 @@ def set_retriever_attributes(attrs: MutableMapping[str, AttributeValue], span: R def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: WorkflowSpan | AgentSpan) -> None: - attrs["gen_ai.input.messages"] = _content_value(span.input) + attrs["gen_ai.input.messages"] = _input_messages(span.input) if span.output is not None: - attrs["gen_ai.output.messages"] = _content_value(span.output) + attrs["gen_ai.output.messages"] = _output_messages(span.output) def set_workflow_attributes(attrs: MutableMapping[str, AttributeValue], span: WorkflowSpan) -> None: @@ -248,9 +380,9 @@ def set_agent_attributes(attrs: MutableMapping[str, AttributeValue], span: Agent def _set_generic_content(attrs: MutableMapping[str, AttributeValue], span: BaseStep) -> None: if span.input is not None: - attrs["gen_ai.input.messages"] = _content_value(span.input) + attrs["gen_ai.input.messages"] = _input_messages(span.input) if span.output is not None: - attrs["gen_ai.output.messages"] = _content_value(span.output) + attrs["gen_ai.output.messages"] = _output_messages(span.output) def build_span_attributes(span: BaseStep, session_id: str | None = None) -> dict[str, AttributeValue]: diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 389cae20..1f2c2cdf 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -3,6 +3,7 @@ import pytest +from galileo_core.schemas.logging.llm import Message, MessageRole, ToolCall, ToolCallFunction from galileo_core.schemas.logging.span import ( AgentSpan, AgentType, @@ -12,6 +13,7 @@ ToolSpan, WorkflowSpan, ) +from galileo_core.schemas.shared.content_parts import FileContentPart from galileo_core.schemas.shared.document import Document from splunk_ao.converter.attribute_mapping import ( CONTENT_ALIAS_BY_GEN_AI, @@ -20,6 +22,14 @@ normalize_attributes_for_export, ) from splunk_ao.logger.control import ControlResult, ControlSpan +from splunk_ao.schema import DataContentBlock, LoggedLlmSpan, LoggedMessage, TextContentBlock + + +def _text_message(role: str, content: str, *, finish_reason: str | None = None) -> dict: + message = {"role": role, "parts": [{"type": "text", "content": content}]} + if finish_reason is not None: + message["finish_reason"] = finish_reason + return message def test_llm_mapping_covers_content_request_response_usage_and_units() -> None: @@ -45,8 +55,8 @@ def test_llm_mapping_covers_content_request_response_usage_and_units() -> None: attrs = build_span_attributes(span) assert attrs["gen_ai.operation.name"] == "chat" - assert json.loads(attrs["gen_ai.input.messages"])[0]["content"] == "prompt" - assert json.loads(attrs["gen_ai.output.messages"])["content"] == "answer" + assert json.loads(attrs["gen_ai.input.messages"]) == [_text_message("user", "prompt")] + assert json.loads(attrs["gen_ai.output.messages"]) == [_text_message("assistant", "answer", finish_reason="stop")] assert attrs["gen_ai.request.model"] == "gpt-5-nano" assert attrs["gen_ai.request.temperature"] == 0.0 assert attrs["gen_ai.response.finish_reasons"] == ("stop",) @@ -60,6 +70,114 @@ def test_llm_mapping_covers_content_request_response_usage_and_units() -> None: assert json.loads(attrs["gen_ai.tool.definitions"])[0]["name"] == "search" +def test_llm_output_uses_unknown_when_finish_reason_is_absent() -> None: + attrs = build_span_attributes(LlmSpan(input="prompt", output="answer")) + + assert json.loads(attrs["gen_ai.output.messages"]) == [ + _text_message("assistant", "answer", finish_reason="unknown") + ] + + +def test_llm_messages_preserve_tool_calls_and_tool_responses() -> None: + span = LlmSpan( + input=[ + Message( + role=MessageRole.assistant, + content="", + tool_calls=[ + ToolCall(id="call-1", function=ToolCallFunction(name="weather", arguments='{"city":"Paris"}')) + ], + ), + Message(role=MessageRole.tool, content='{"temperature":21}', tool_call_id="call-1"), + ], + output="It is 21 degrees.", + finish_reason="stop", + ) + + messages = json.loads(build_span_attributes(span)["gen_ai.input.messages"]) + + assert messages == [ + { + "role": "assistant", + "parts": [{"type": "tool_call", "id": "call-1", "name": "weather", "arguments": {"city": "Paris"}}], + }, + {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call-1", "response": {"temperature": 21}}]}, + ] + + +def test_llm_content_parts_preserve_available_nonstandard_fields() -> None: + file_id = uuid4() + stored_file_span = LlmSpan( + input=[Message(role=MessageRole.user, content=[FileContentPart(file_id=file_id)])], output="answer" + ) + ingest_span = LoggedLlmSpan( + input=[ + LoggedMessage( + role=MessageRole.user, + content=[ + TextContentBlock(text="inspect this", index=2, metadata={"language": "en"}), + DataContentBlock( + modality="document", + mime_type="application/pdf", + base64="ZG9jdW1lbnQ=", + index=3, + metadata={"source": "upload"}, + ), + ], + ) + ], + output="answer", + ) + + stored_parts = json.loads(build_span_attributes(stored_file_span)["gen_ai.input.messages"])[0]["parts"] + ingest_parts = json.loads(build_span_attributes(ingest_span)["gen_ai.input.messages"])[0]["parts"] + + assert stored_parts == [{"type": "file", "file_id": str(file_id)}] + assert ingest_parts == [ + {"type": "text", "content": "inspect this", "index": 2, "metadata": {"language": "en"}}, + { + "type": "data", + "modality": "document", + "mime_type": "application/pdf", + "base64": "ZG9jdW1lbnQ=", + "index": 3, + "metadata": {"source": "upload"}, + }, + ] + assert "modality" not in stored_parts[0] + assert "content" not in stored_parts[0] + + +def test_llm_tool_definitions_flatten_openai_functions_and_preserve_flat_definitions() -> None: + span = LlmSpan( + input="prompt", + output="answer", + tools=[ + { + "type": "function", + "function": { + "name": "weather", + "description": "Get the weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, + }, + {"type": "function", "name": "search", "custom": "preserved"}, + ], + ) + + definitions = json.loads(build_span_attributes(span)["gen_ai.tool.definitions"]) + + assert definitions == [ + { + "type": "function", + "name": "weather", + "description": "Get the weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, + {"type": "function", "name": "search", "custom": "preserved"}, + ] + + def test_common_mapping_uses_canonical_keys_and_omits_external_id() -> None: session_id = uuid4() span = ToolSpan( @@ -81,8 +199,8 @@ def test_common_mapping_uses_canonical_keys_and_omits_external_id() -> None: assert attrs["gen_ai.operation.name"] == "execute_tool" assert attrs["gen_ai.tool.name"] == "search" - assert attrs["gen_ai.tool.call.arguments"] == '{"q":"x"}' - assert attrs["gen_ai.tool.call.result"] == "result" + assert json.loads(attrs["gen_ai.tool.call.arguments"]) == {"q": "x"} + assert json.loads(attrs["gen_ai.tool.call.result"]) == {"value": "result"} assert attrs["gen_ai.conversation.id"] == str(session_id) assert json.loads(attrs["splunk_ao.metadata"]) == {"team": "checkout"} assert attrs["splunk_ao.tags"] == ("production",) @@ -110,7 +228,7 @@ def test_retriever_mapping_uses_query_and_documents() -> None: assert attrs["gen_ai.operation.name"] == "retrieval" assert attrs["gen_ai.retrieval.query.text"] == "what is RAG?" - assert json.loads(attrs["gen_ai.retrieval.documents"])[0]["content"] == "doc" + assert json.loads(attrs["gen_ai.retrieval.documents"]) == [{"content": "doc", "metadata": {"source": "kb"}}] assert attrs["splunk_ao.retrieval.documents.count"] == 1 assert attrs["db.operation"] == "search" assert "gen_ai.output.messages" not in attrs @@ -140,22 +258,25 @@ def test_orchestration_mapping(span, operation_key: str, operation: str, name_ke assert attrs[operation_key] == operation assert attrs[name_key] == name - assert attrs["gen_ai.input.messages"] == "question" - assert attrs["gen_ai.output.messages"] == "answer" + assert json.loads(attrs["gen_ai.input.messages"]) == [_text_message("user", "question")] + assert json.loads(attrs["gen_ai.output.messages"]) == [ + _text_message("assistant", "answer", finish_reason="unknown") + ] def test_control_mapping_preserves_structured_content() -> None: span = ControlSpan( - name="guardrail", - input={"text": "question"}, - output=ControlResult(action="observe", matched=True, confidence=0.9), + name="guardrail", input="question", output=ControlResult(action="observe", matched=True, confidence=0.9) ) attrs = build_span_attributes(span) + output = json.loads(attrs["gen_ai.output.messages"]) assert attrs["splunk_ao.operation.name"] == "control" - assert json.loads(attrs["gen_ai.input.messages"]) == {"text": "question"} - assert json.loads(attrs["gen_ai.output.messages"])["matched"] is True + assert json.loads(attrs["gen_ai.input.messages"]) == [_text_message("user", "question")] + assert output[0]["role"] == "assistant" + assert output[0]["finish_reason"] == "unknown" + assert json.loads(output[0]["parts"][0]["content"])["matched"] is True def test_normalizer_duplicates_ordinary_attributes_and_relocates_all_content() -> None: @@ -199,7 +320,11 @@ def test_normalizer_converts_first_chunk_seconds_to_splunk_nanoseconds() -> None def test_normalizer_is_idempotent() -> None: - source = {"gen_ai.operation.name": "chat", "gen_ai.input.messages": "input-json", "custom.attribute": "unchanged"} + source = { + "gen_ai.operation.name": "chat", + "gen_ai.input.messages": json.dumps([_text_message("user", "question")]), + "custom.attribute": "unchanged", + } once = normalize_attributes_for_export(source) @@ -209,7 +334,7 @@ def test_normalizer_is_idempotent() -> None: def test_normalizer_can_be_disabled_for_developer_comparison() -> None: source = { "gen_ai.request.model": "gpt-4o", - "gen_ai.input.messages": "input-json", + "gen_ai.input.messages": json.dumps([_text_message("user", "question")]), "splunk_ao.system": "source-value", } diff --git a/tests/test_otel.py b/tests/test_otel.py index 69826847..d7d046a8 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -277,8 +277,8 @@ def test_tool_span_with_all_fields(self): assert attrs["gen_ai.operation.name"] == "execute_tool" assert attrs["gen_ai.tool.name"] == "test-tool" - assert attrs["gen_ai.tool.call.arguments"] == "tool input data" - assert attrs["gen_ai.tool.call.result"] == "tool output result" + assert json.loads(attrs["gen_ai.tool.call.arguments"]) == {"value": "tool input data"} + assert json.loads(attrs["gen_ai.tool.call.result"]) == {"value": "tool output result"} assert attrs["gen_ai.tool.call.id"] == "call-123" assert "gen_ai.input.messages" not in attrs assert "gen_ai.output.messages" not in attrs @@ -291,7 +291,7 @@ def test_tool_span_with_only_input(self): assert attrs["gen_ai.operation.name"] == "execute_tool" assert attrs["gen_ai.tool.name"] == "test-tool" - assert attrs["gen_ai.tool.call.arguments"] == "tool input only" + assert json.loads(attrs["gen_ai.tool.call.arguments"]) == {"value": "tool input only"} assert "gen_ai.tool.call.result" not in attrs assert "gen_ai.tool.call.id" not in attrs @@ -305,8 +305,8 @@ def test_tool_span_with_output_no_tool_call_id(self): assert attrs["gen_ai.operation.name"] == "execute_tool" assert attrs["gen_ai.tool.name"] == "test-tool" - assert attrs["gen_ai.tool.call.arguments"] == "tool input" - assert attrs["gen_ai.tool.call.result"] == "tool output" + assert json.loads(attrs["gen_ai.tool.call.arguments"]) == {"value": "tool input"} + assert json.loads(attrs["gen_ai.tool.call.result"]) == {"value": "tool output"} assert "gen_ai.tool.call.id" not in attrs @@ -347,8 +347,8 @@ def test_start_splunk_ao_span_dispatches_tool_span(self): assert "gen_ai.system" not in calls assert calls["gen_ai.operation.name"] == "execute_tool" assert calls["gen_ai.tool.name"] == "my-tool" - assert calls["gen_ai.tool.call.arguments"] == "tool input data" - assert calls["gen_ai.tool.call.result"] == "tool output result" + assert json.loads(calls["gen_ai.tool.call.arguments"]) == {"value": "tool input data"} + assert json.loads(calls["gen_ai.tool.call.result"]) == {"value": "tool output result"} assert calls["gen_ai.tool.call.id"] == "call-789" def test_start_splunk_ao_span_tool_span_with_none_output(self): @@ -372,7 +372,7 @@ def test_start_splunk_ao_span_tool_span_with_none_output(self): assert "gen_ai.system" not in calls assert calls["gen_ai.operation.name"] == "execute_tool" assert calls["gen_ai.tool.name"] == "minimal-tool" - assert calls["gen_ai.tool.call.arguments"] == "just input" + assert json.loads(calls["gen_ai.tool.call.arguments"]) == {"value": "just input"} assert "gen_ai.tool.call.result" not in calls assert "gen_ai.tool.call.id" not in calls @@ -435,8 +435,12 @@ def test_workflow_span_with_string_input_output(self): assert attrs["gen_ai.operation.name"] == "invoke_workflow" assert attrs["gen_ai.workflow.name"] == "test-workflow" - assert attrs["gen_ai.input.messages"] == "input text" - assert attrs["gen_ai.output.messages"] == "output text" + assert json.loads(attrs["gen_ai.input.messages"]) == [ + {"role": "user", "parts": [{"type": "text", "content": "input text"}]} + ] + assert json.loads(attrs["gen_ai.output.messages"]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "output text"}], "finish_reason": "unknown"} + ] def test_workflow_span_with_message_input_output(self): """Test WorkflowSpan with Message input and output.""" @@ -445,8 +449,12 @@ def test_workflow_span_with_message_input_output(self): workflow_span = WorkflowSpan(name="test-workflow", input=[input_msg], output=input_msg, status_code=200) attrs = build_span_attributes(workflow_span) - assert json.loads(attrs["gen_ai.input.messages"])[0]["content"] == "user question" - assert json.loads(attrs["gen_ai.output.messages"])["content"] == "user question" + assert json.loads(attrs["gen_ai.input.messages"]) == [ + {"role": "user", "parts": [{"type": "text", "content": "user question"}]} + ] + assert json.loads(attrs["gen_ai.output.messages"]) == [ + {"role": "user", "parts": [{"type": "text", "content": "user question"}], "finish_reason": "unknown"} + ] def test_workflow_span_with_document_sequence_output(self): """Test WorkflowSpan with Document sequence output.""" @@ -463,10 +471,9 @@ def test_workflow_span_with_document_sequence_output(self): ) attrs = build_span_attributes(workflow_span) - assert [document["content"] for document in json.loads(attrs["gen_ai.output.messages"])] == [ - "doc1 content", - "doc2 content", - ] + output = json.loads(attrs["gen_ai.output.messages"]) + documents = json.loads(output[0]["parts"][0]["content"]) + assert [document["content"] for document in documents] == ["doc1 content", "doc2 content"] def test_workflow_span_with_none_output(self): """Test WorkflowSpan with None output (should not set output attribute).""" @@ -474,7 +481,9 @@ def test_workflow_span_with_none_output(self): workflow_span = WorkflowSpan(name="test-workflow", input="input text", output=None, status_code=200) attrs = build_span_attributes(workflow_span) - assert attrs["gen_ai.input.messages"] == "input text" + assert json.loads(attrs["gen_ai.input.messages"]) == [ + {"role": "user", "parts": [{"type": "text", "content": "input text"}]} + ] assert "gen_ai.output.messages" not in attrs def test_workflow_span_in_start_splunk_ao_span(self): From e5396a3540d0127749c8d1ac1df20edb22a47671 Mon Sep 17 00:00:00 2001 From: pradystar Date: Fri, 24 Jul 2026 15:48:36 -0700 Subject: [PATCH 04/10] address attribute normalization review findings --- splunk-ao-a2a/src/splunk_ao_a2a/_constants.py | 10 ++-- .../tests/test_splunk_ao_compatibility.py | 3 +- src/splunk_ao/converter/attribute_mapping.py | 3 +- src/splunk_ao/otel.py | 7 ++- tests/test_attribute_mapping.py | 15 ++++++ tests/test_otel.py | 50 +++++++++++++++++++ 6 files changed, 79 insertions(+), 9 deletions(-) diff --git a/splunk-ao-a2a/src/splunk_ao_a2a/_constants.py b/splunk-ao-a2a/src/splunk_ao_a2a/_constants.py index efdb943d..eed3a454 100644 --- a/splunk-ao-a2a/src/splunk_ao_a2a/_constants.py +++ b/splunk-ao-a2a/src/splunk_ao_a2a/_constants.py @@ -10,11 +10,11 @@ SPLUNK_AO_OBSERVE_KEY = "splunk_ao_observe" AGNTCY_OBSERVE_KEY = "observe" # compatibility with AGNTCY Observe SDK -# A2A span attribute keys -A2A_TASK_ID = "splunk_ao.a2a.task.id" -A2A_CONTEXT_ID = "splunk_ao.a2a.context_id" -A2A_RPC_METHOD = "splunk_ao.a2a.rpc.method" -A2A_TASK_STATE = "splunk_ao.a2a.task.state" +# A2A protocol span attributes expected by the ingest API +A2A_TASK_ID = "a2a.task.id" +A2A_CONTEXT_ID = "a2a.context_id" +A2A_RPC_METHOD = "a2a.rpc.method" +A2A_TASK_STATE = "a2a.task.state" # OTel GenAI semantic convention attributes — span type determination GENAI_OPERATION_NAME = "gen_ai.operation.name" diff --git a/splunk-ao-a2a/tests/test_splunk_ao_compatibility.py b/splunk-ao-a2a/tests/test_splunk_ao_compatibility.py index 330078b9..412f0841 100644 --- a/splunk-ao-a2a/tests/test_splunk_ao_compatibility.py +++ b/splunk-ao-a2a/tests/test_splunk_ao_compatibility.py @@ -75,7 +75,8 @@ def exporter_factory(**kwargs: object) -> RecordingExporter: assert exported.instrumentation_scope.name == INSTRUMENTOR_NAME assert "gen_ai.system" not in exported.attributes assert exported.attributes["splunk_ao.system"] == "splunk_ao_python" - assert exported.attributes["splunk_ao.a2a.rpc.method"] == "SendMessage" + assert exported.attributes["a2a.rpc.method"] == "SendMessage" + assert "splunk_ao.a2a.rpc.method" not in exported.attributes assert exported.attributes["gen_ai.conversation.id"] == "context-id" assert exported.attributes["splunk_ao.session.id"] == "context-id" assert exported.attributes["gen_ai.operation.name"] == "invoke_agent" diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 3381efe6..8b570cf0 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -27,7 +27,6 @@ } SPLUNK_ALIAS_BY_GEN_AI: Mapping[str, str] = { - "gen_ai.system": "splunk_ao.provider.name", "gen_ai.operation.name": "splunk_ao.operation.name", "gen_ai.conversation.id": "splunk_ao.session.id", "gen_ai.workflow.name": "splunk_ao.workflow.name", @@ -428,6 +427,8 @@ def normalize_attributes_for_export( for source_key, destination_key in SPLUNK_ALIAS_BY_GEN_AI.items(): if source_key in attrs: + if destination_key == "splunk_ao.llm.time_to_first_token_ns" and destination_key in attrs: + continue result[destination_key] = _alias_value(source_key, attrs[source_key]) for source_key in CONTENT_ALIAS_BY_GEN_AI: diff --git a/src/splunk_ao/otel.py b/src/splunk_ao/otel.py index 3c81ff94..863e4f12 100644 --- a/src/splunk_ao/otel.py +++ b/src/splunk_ao/otel.py @@ -320,5 +320,8 @@ def start_splunk_ao_span(galileo_span: GalileoSpan) -> Generator[trace.Span, Any try: yield span finally: - for key, value in build_span_attributes(galileo_span, _session_id_context.get(None)).items(): - span.set_attribute(key, value) + try: + for key, value in build_span_attributes(galileo_span, _session_id_context.get(None)).items(): + span.set_attribute(key, value) + except Exception: + logger.warning("Failed to finalize Splunk AO span attributes", exc_info=True) diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 1f2c2cdf..94ccbbca 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -312,6 +312,13 @@ def test_normalizer_gen_ai_wins_collisions_and_sdk_marker_is_authoritative() -> assert result["splunk_ao.system"] == "splunk_ao_python" +def test_normalizer_does_not_use_deprecated_gen_ai_system_as_provider() -> None: + result = normalize_attributes_for_export({"gen_ai.system": "legacy-provider"}) + + assert result["gen_ai.system"] == "legacy-provider" + assert "splunk_ao.provider.name" not in result + + def test_normalizer_converts_first_chunk_seconds_to_splunk_nanoseconds() -> None: result = normalize_attributes_for_export({"gen_ai.response.time_to_first_chunk": 0.125}) @@ -319,6 +326,14 @@ def test_normalizer_converts_first_chunk_seconds_to_splunk_nanoseconds() -> None assert result["splunk_ao.llm.time_to_first_token_ns"] == 125_000_000 +def test_normalizer_preserves_exact_source_nanoseconds() -> None: + result = normalize_attributes_for_export( + {"gen_ai.response.time_to_first_chunk": 0.123456789, "splunk_ao.llm.time_to_first_token_ns": 123_456_789} + ) + + assert result["splunk_ao.llm.time_to_first_token_ns"] == 123_456_789 + + def test_normalizer_is_idempotent() -> None: source = { "gen_ai.operation.name": "chat", diff --git a/tests/test_otel.py b/tests/test_otel.py index d7d046a8..89931423 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -423,6 +423,56 @@ def test_start_span_applies_canonical_attributes_when_body_raises(self): assert calls["gen_ai.operation.name"] == "execute_tool" assert calls["gen_ai.tool.name"] == "search" + @patch("splunk_ao.otel.logger.warning") + @patch("splunk_ao.otel.build_span_attributes", side_effect=ValueError("invalid partial span")) + def test_start_span_finalization_does_not_mask_body_exception(self, _mock_build, mock_warning): + galileo_span = ToolSpan(name="search", input="query", output="result") + mock_tracer = Mock() + mock_tracer.start_as_current_span.return_value.__enter__ = Mock(return_value=Mock()) + mock_tracer.start_as_current_span.return_value.__exit__ = Mock(return_value=False) + mock_provider = Mock() + mock_provider.get_tracer.return_value = mock_tracer + _TRACE_PROVIDER_CONTEXT_VAR.set(mock_provider) + + with pytest.raises(RuntimeError, match="user failure"), start_splunk_ao_span(galileo_span): + raise RuntimeError("user failure") + + mock_warning.assert_called_once_with("Failed to finalize Splunk AO span attributes", exc_info=True) + + @patch("splunk_ao.otel.logger.warning") + @patch("splunk_ao.otel.build_span_attributes", side_effect=ValueError("invalid span")) + def test_start_span_finalization_failure_does_not_fail_successful_body(self, _mock_build, mock_warning): + galileo_span = ToolSpan(name="search", input="query", output="result") + mock_tracer = Mock() + mock_tracer.start_as_current_span.return_value.__enter__ = Mock(return_value=Mock()) + mock_tracer.start_as_current_span.return_value.__exit__ = Mock(return_value=False) + mock_provider = Mock() + mock_provider.get_tracer.return_value = mock_tracer + _TRACE_PROVIDER_CONTEXT_VAR.set(mock_provider) + + with start_splunk_ao_span(galileo_span): + pass + + mock_warning.assert_called_once_with("Failed to finalize Splunk AO span attributes", exc_info=True) + + @patch("splunk_ao.otel.logger.warning") + @patch("splunk_ao.otel.build_span_attributes", return_value={"gen_ai.operation.name": "execute_tool"}) + def test_start_span_contains_attribute_write_failure(self, _mock_build, mock_warning): + galileo_span = ToolSpan(name="search", input="query", output="result") + mock_otel_span = Mock() + mock_otel_span.set_attribute.side_effect = ValueError("attribute rejected") + mock_tracer = Mock() + mock_tracer.start_as_current_span.return_value.__enter__ = Mock(return_value=mock_otel_span) + mock_tracer.start_as_current_span.return_value.__exit__ = Mock(return_value=False) + mock_provider = Mock() + mock_provider.get_tracer.return_value = mock_tracer + _TRACE_PROVIDER_CONTEXT_VAR.set(mock_provider) + + with start_splunk_ao_span(galileo_span): + pass + + mock_warning.assert_called_once_with("Failed to finalize Splunk AO span attributes", exc_info=True) + class TestWorkflowSpanAttributes: """Test suite for WorkflowSpan OpenTelemetry attribute mapping.""" From 5b777ed7c4ad79f1e0e17b5eda1bbedc8785037c Mon Sep 17 00:00:00 2001 From: pradystar Date: Fri, 24 Jul 2026 18:06:16 -0700 Subject: [PATCH 05/10] ix(otel): extract schema-valid messages --- src/splunk_ao/converter/attribute_mapping.py | 96 +++++++++-- tests/test_attribute_mapping.py | 159 ++++++++++++++++++- 2 files changed, 241 insertions(+), 14 deletions(-) diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 8b570cf0..7b09c6d6 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -100,15 +100,26 @@ def _mapping_value(value: Any) -> dict[str, Any] | None: return None -def _parse_json_value(value: Any) -> Any: +def _mapping_view(value: Any) -> Mapping[str, Any] | None: + if isinstance(value, BaseModel): + return value.model_dump(mode="json", exclude_none=True) + return value if isinstance(value, Mapping) else None + + +def _parse_json_string(value: Any) -> Any: if not isinstance(value, str): - return _json_compatible(value) + return value try: return json.loads(value) except (TypeError, ValueError): return value +def _parse_json_value(value: Any) -> Any: + parsed = _parse_json_string(value) + return parsed if isinstance(value, str) else _json_compatible(parsed) + + def _text_part(value: Any) -> dict[str, Any]: content = value if isinstance(value, str) else _json_string(value) return {"type": "text", "content": content} @@ -153,17 +164,20 @@ def _tool_call_part(value: Any) -> dict[str, Any]: def _mapped_message(source: dict[str, Any], default_role: str) -> dict[str, Any]: role = _json_compatible(source.pop("role")) - content = source.pop("content", "") + content = source.pop("content", None) + source_parts = source.pop("parts", None) tool_call_id = source.pop("tool_call_id", None) tool_calls = source.pop("tool_calls", None) - if role == "tool": + if source_parts is not None: + parts = _content_parts(source_parts) + elif role == "tool": response = {"type": "tool_call_response", "response": _parse_json_value(content)} if tool_call_id is not None: response["id"] = str(tool_call_id) parts = [response] else: - parts = [] if content == "" and tool_calls else _content_parts(content) + parts = [] if content in (None, "") and tool_calls else _content_parts("" if content is None else content) if tool_calls: parts.extend(_tool_call_part(tool_call) for tool_call in tool_calls) @@ -190,16 +204,61 @@ def _message_sequence(value: Any, default_role: str) -> list[dict[str, Any]]: return [_message(value, default_role)] -def _input_messages(value: Any) -> str: - return _json_string(_message_sequence(value, "user")) +def _message_container(value: Any) -> tuple[Any | None, bool]: + parsed = _parse_json_string(value) + source = _mapping_view(parsed) + if source is not None: + update = _mapping_view(_parse_json_string(source.get("update"))) + for container in (update, source): + if container is None or "messages" not in container: + continue + messages = _parse_json_string(container["messages"]) + if _is_message_sequence(messages): + return messages, container is source -def _output_messages(value: Any, finish_reason: str | None = None) -> str: - messages = _message_sequence(value, "assistant") + if "role" in source or "type" in source: + return parsed, False + return None, False + + if isinstance(parsed, Sequence) and not isinstance(parsed, str | bytes | bytearray): + if _is_message_sequence(parsed) or _is_content_part_sequence(parsed): + return parsed, False + return (parsed, False) if not isinstance(value, str | bytes | bytearray) else (None, False) + + return parsed, False + + +def _is_message_sequence(value: Any) -> bool: + if not isinstance(value, Sequence) or isinstance(value, str | bytes | bytearray): + message = _mapping_view(value) + return message is not None and "role" in message + return not value or all((message := _mapping_view(item)) is not None and "role" in message for item in value) + + +def _is_content_part_sequence(value: Any) -> bool: + return bool(value) and all((part := _mapping_view(item)) is not None and "type" in part for item in value) + + +def _orchestration_messages(value: Any, default_role: str) -> tuple[list[dict[str, Any]] | None, bool]: + container, full_history = _message_container(value) + messages = None if container is None else _message_sequence(container, default_role) + return messages, full_history + + +def _with_finish_reasons(messages: list[dict[str, Any]], finish_reason: str | None = None) -> list[dict[str, Any]]: for message in messages: source_finish_reason = message.get("finish_reason") message["finish_reason"] = finish_reason or source_finish_reason or "unknown" - return _json_string(messages) + return messages + + +def _input_messages(value: Any) -> str: + return _json_string(_message_sequence(value, "user")) + + +def _output_messages(value: Any, finish_reason: str | None = None) -> str: + return _json_string(_with_finish_reasons(_message_sequence(value, "assistant"), finish_reason)) def _tool_definitions(value: Sequence[Any]) -> str: @@ -357,9 +416,20 @@ def set_retriever_attributes(attrs: MutableMapping[str, AttributeValue], span: R def _set_orchestration_content(attrs: MutableMapping[str, AttributeValue], span: WorkflowSpan | AgentSpan) -> None: - attrs["gen_ai.input.messages"] = _input_messages(span.input) - if span.output is not None: - attrs["gen_ai.output.messages"] = _output_messages(span.output) + input_messages, _ = _orchestration_messages(span.input, "user") + if input_messages is not None: + attrs["gen_ai.input.messages"] = _json_string(input_messages) + + if span.output is None: + return + + output_messages, full_history = _orchestration_messages(span.output, "assistant") + if output_messages is None: + return + if full_history and input_messages is not None and output_messages[: len(input_messages)] == input_messages: + output_messages = output_messages[len(input_messages) :] + if output_messages: + attrs["gen_ai.output.messages"] = _json_string(_with_finish_reasons(output_messages)) def set_workflow_attributes(attrs: MutableMapping[str, AttributeValue], span: WorkflowSpan) -> None: diff --git a/tests/test_attribute_mapping.py b/tests/test_attribute_mapping.py index 94ccbbca..3e4cc859 100644 --- a/tests/test_attribute_mapping.py +++ b/tests/test_attribute_mapping.py @@ -13,7 +13,7 @@ ToolSpan, WorkflowSpan, ) -from galileo_core.schemas.shared.content_parts import FileContentPart +from galileo_core.schemas.shared.content_parts import FileContentPart, TextContentPart from galileo_core.schemas.shared.document import Document from splunk_ao.converter.attribute_mapping import ( CONTENT_ALIAS_BY_GEN_AI, @@ -264,6 +264,163 @@ def test_orchestration_mapping(span, operation_key: str, operation: str, name_ke ] +def test_orchestration_extracts_serialized_langgraph_messages_with_multimodal_parts() -> None: + file_id = uuid4() + span = WorkflowSpan( + name="travel-planner", + input=json.dumps( + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Plan this trip"}, + {"type": "image", "url": "https://example.com/map.png"}, + {"type": "file", "file_id": str(file_id)}, + ], + } + ], + "destination": "Paris", + } + ), + output=json.dumps( + { + "update": { + "messages": [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here is the plan"}, + {"type": "data", "modality": "audio", "base64": "YXVkaW8="}, + ], + } + ] + } + } + ), + ) + + attrs = build_span_attributes(span) + + assert json.loads(attrs["gen_ai.input.messages"]) == [ + { + "role": "user", + "parts": [ + {"type": "text", "content": "Plan this trip"}, + {"type": "image", "url": "https://example.com/map.png"}, + {"type": "file", "file_id": str(file_id)}, + ], + } + ] + assert json.loads(attrs["gen_ai.output.messages"]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": "Here is the plan"}, + {"type": "data", "modality": "audio", "base64": "YXVkaW8="}, + ], + "finish_reason": "unknown", + } + ] + + +def test_orchestration_accepts_native_content_part_sequences() -> None: + file_id = uuid4() + span = AgentSpan( + name="multimodal-agent", + agent_type=AgentType.planner, + input=[TextContentPart(text="Inspect this"), FileContentPart(file_id=file_id)], + output=[FileContentPart(file_id=file_id)], + ) + + attrs = build_span_attributes(span) + + assert json.loads(attrs["gen_ai.input.messages"]) == [ + { + "role": "user", + "parts": [{"type": "text", "content": "Inspect this"}, {"type": "file", "file_id": str(file_id)}], + } + ] + assert json.loads(attrs["gen_ai.output.messages"]) == [ + {"role": "assistant", "parts": [{"type": "file", "file_id": str(file_id)}], "finish_reason": "unknown"} + ] + + +def test_orchestration_output_omits_repeated_input_history() -> None: + user_message = {"role": "user", "content": "Plan a trip"} + assistant_message = {"role": "assistant", "content": "Where would you like to go?"} + span = AgentSpan( + name="planner", + agent_type=AgentType.planner, + input=json.dumps({"messages": [user_message]}), + output=json.dumps({"messages": [user_message, assistant_message]}), + ) + + attrs = build_span_attributes(span) + + assert json.loads(attrs["gen_ai.input.messages"]) == [_text_message("user", "Plan a trip")] + assert json.loads(attrs["gen_ai.output.messages"]) == [ + _text_message("assistant", "Where would you like to go?", finish_reason="unknown") + ] + + +def test_orchestration_preserves_schema_valid_parts_and_tool_calls() -> None: + span = WorkflowSpan( + name="tool-workflow", + input=json.dumps( + { + "messages": [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Checking weather"}], + "tool_calls": [ + {"id": "call-1", "function": {"name": "weather", "arguments": '{"city":"Paris"}'}} + ], + }, + {"role": "tool", "content": '{"temperature":21}', "tool_call_id": "call-1"}, + ] + } + ), + ) + + messages = json.loads(build_span_attributes(span)["gen_ai.input.messages"]) + + assert messages == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": "Checking weather"}, + {"type": "tool_call", "id": "call-1", "name": "weather", "arguments": {"city": "Paris"}}, + ], + }, + {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call-1", "response": {"temperature": 21}}]}, + ] + + +def test_orchestration_does_not_label_arbitrary_state_as_messages() -> None: + span = WorkflowSpan( + name="state-machine", + input=json.dumps({"current_agent": "coordinator", "travellers": 2}), + output=json.dumps({"next_agent": "flight_specialist"}), + ) + + attrs = build_span_attributes(span) + + assert "gen_ai.input.messages" not in attrs + assert "gen_ai.output.messages" not in attrs + + +def test_orchestration_keeps_non_json_strings_as_text_messages() -> None: + span = WorkflowSpan(name="workflow", input="{not-json", output="plain response") + + attrs = build_span_attributes(span) + + assert json.loads(attrs["gen_ai.input.messages"]) == [_text_message("user", "{not-json")] + assert json.loads(attrs["gen_ai.output.messages"]) == [ + _text_message("assistant", "plain response", finish_reason="unknown") + ] + + def test_control_mapping_preserves_structured_content() -> None: span = ControlSpan( name="guardrail", input="question", output=ControlResult(action="observe", matched=True, confidence=0.9) From 80e0d5f4539a48505ca0a63feeea1b3dc057de12 Mon Sep 17 00:00:00 2001 From: pradystar Date: Wed, 22 Jul 2026 09:05:44 -0700 Subject: [PATCH 06/10] feat(otel): add proprietary span converter --- src/splunk_ao/converter/__init__.py | 2 + src/splunk_ao/converter/span_converter.py | 106 ++++++++ tests/test_span_converter.py | 291 ++++++++++++++++++++++ 3 files changed, 399 insertions(+) create mode 100644 src/splunk_ao/converter/span_converter.py create mode 100644 tests/test_span_converter.py diff --git a/src/splunk_ao/converter/__init__.py b/src/splunk_ao/converter/__init__.py index 654ae5dd..e33f7569 100644 --- a/src/splunk_ao/converter/__init__.py +++ b/src/splunk_ao/converter/__init__.py @@ -6,10 +6,12 @@ build_span_attributes, normalize_attributes_for_export, ) +from splunk_ao.converter.span_converter import SpanConverter __all__ = [ "CONTENT_ALIAS_BY_GEN_AI", "SPLUNK_ALIAS_BY_GEN_AI", + "SpanConverter", "build_span_attributes", "normalize_attributes_for_export", ] diff --git a/src/splunk_ao/converter/span_converter.py b/src/splunk_ao/converter/span_converter.py new file mode 100644 index 00000000..2f4cefa7 --- /dev/null +++ b/src/splunk_ao/converter/span_converter.py @@ -0,0 +1,106 @@ +"""Convert completed Galileo spans to OpenTelemetry spans.""" + +from __future__ import annotations + +import time +from datetime import UTC, datetime + +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from opentelemetry.trace import SpanContext, SpanKind, Status, StatusCode + +from galileo_core.schemas.logging.step import BaseStep, StepType +from splunk_ao.converter.attribute_mapping import build_span_attributes +from splunk_ao.utils.headers_data import get_package_version + +_EPOCH = datetime(1970, 1, 1, tzinfo=UTC) +_INSTRUMENTATION_SCOPE = InstrumentationScope(name="splunk_ao", version=get_package_version()) + +_NAME_PARTS: dict[StepType, tuple[str, str]] = { + StepType.llm: ("chat", "model"), + StepType.tool: ("execute_tool", "name"), + StepType.retriever: ("retrieval", "name"), + StepType.workflow: ("invoke_workflow", "name"), + StepType.agent: ("invoke_agent", "name"), + StepType.trace: ("", "name"), + StepType.control: ("", "name"), +} + +_KIND_BY_STEP_TYPE = { + step_type: SpanKind.CLIENT if step_type is StepType.llm else SpanKind.INTERNAL for step_type in _NAME_PARTS +} + + +def _step_type(span: BaseStep) -> StepType: + raw_type = getattr(span, "type", None) + try: + step_type = StepType(raw_type) + except (TypeError, ValueError) as exc: + raise TypeError(f"Unsupported step type: {raw_type}") from exc + + if step_type not in _NAME_PARTS: + raise TypeError(f"Unsupported step type: {step_type.value}") + return step_type + + +def _span_name(span: BaseStep, step_type: StepType) -> str: + prefix, field_name = _NAME_PARTS[step_type] + detail = getattr(span, field_name, None) + return " ".join(part for part in (prefix, str(detail).strip() if detail is not None else "") if part) + + +def _to_unix_ns(value: datetime) -> int: + if value.tzinfo is None: + value = value.replace(tzinfo=UTC) + delta = value.astimezone(UTC) - _EPOCH + return ((delta.days * 86_400 + delta.seconds) * 1_000_000_000) + delta.microseconds * 1_000 + + +def _span_status(span: BaseStep) -> Status: + exception = getattr(span, "exception", None) + status_code = getattr(span, "status_code", None) + if exception is None and (status_code is None or status_code < 400): + return Status(StatusCode.UNSET) + return Status(StatusCode.ERROR, str(exception) if exception is not None else "") + + +def _end_time_ns(span: BaseStep, start_time_ns: int, end_time_ns: int | None) -> int: + duration_ns = span.metrics.duration_ns + if duration_ns is not None: + return start_time_ns + duration_ns + if end_time_ns is not None: + return end_time_ns + return time.time_ns() + + +class SpanConverter: + """Convert one completed Galileo span into an OTel readable span.""" + + def convert_span( + self, + span: BaseStep, + span_context: SpanContext, + parent_span_context: SpanContext | None, + session_id: str | None, + resource: Resource, + end_time_ns: int | None = None, + ) -> ReadableSpan: + """Build a readable span using the supplied identity and resource.""" + step_type = _step_type(span) + start_time_ns = _to_unix_ns(span.created_at) + + return ReadableSpan( + name=_span_name(span, step_type), + context=span_context, + parent=parent_span_context, + resource=resource, + attributes=build_span_attributes(span, session_id), + events=(), + links=(), + kind=_KIND_BY_STEP_TYPE[step_type], + instrumentation_scope=_INSTRUMENTATION_SCOPE, + status=_span_status(span), + start_time=start_time_ns, + end_time=_end_time_ns(span, start_time_ns, end_time_ns), + ) diff --git a/tests/test_span_converter.py b/tests/test_span_converter.py new file mode 100644 index 00000000..ca983275 --- /dev/null +++ b/tests/test_span_converter.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import pytest +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.trace import SpanContext, SpanKind, StatusCode, TraceFlags, TraceState + +from galileo_core.schemas.logging.span import AgentSpan, LlmSpan, RetrieverSpan, ToolSpan, WorkflowSpan +from galileo_core.schemas.logging.step import BaseStep, Metrics, StepType +from galileo_core.schemas.logging.trace import Trace +from galileo_core.schemas.shared.document import Document +from splunk_ao.converter import SpanConverter, span_converter +from splunk_ao.converter.attribute_mapping import build_span_attributes +from splunk_ao.logger.control import ControlResult, ControlSpan +from splunk_ao.utils.headers_data import get_package_version + +TRACE_ID = 0x1234567890ABCDEF1234567890ABCDEF +SPAN_ID = 0x1234567890ABCDEF +PARENT_SPAN_ID = 0xFEDCBA0987654321 +CREATED_AT = datetime(2025, 1, 1, tzinfo=UTC) + + +def make_context( + *, + trace_id: int = TRACE_ID, + span_id: int = SPAN_ID, + is_remote: bool = False, + trace_flags: TraceFlags = TraceFlags(TraceFlags.SAMPLED), + trace_state: TraceState | None = None, +) -> SpanContext: + return SpanContext( + trace_id=trace_id, + span_id=span_id, + is_remote=is_remote, + trace_flags=trace_flags, + trace_state=trace_state or TraceState(), + ) + + +def convert( + span: BaseStep, + *, + span_context: SpanContext | None = None, + parent_span_context: SpanContext | None = None, + resource: Resource | None = None, + session_id: str | None = "session-1", + end_time_ns: int | None = None, +) -> ReadableSpan: + return SpanConverter().convert_span( + span, + span_context=span_context or make_context(), + parent_span_context=parent_span_context, + session_id=session_id, + resource=resource or Resource.create({}), + end_time_ns=end_time_ns, + ) + + +def supported_spans() -> list[tuple[BaseStep, str, SpanKind]]: + return [ + (LlmSpan(input="prompt", output="answer", model="gpt-5", created_at=CREATED_AT), "chat gpt-5", SpanKind.CLIENT), + ( + ToolSpan(name="search", input='{"q":"otel"}', output="result", created_at=CREATED_AT), + "execute_tool search", + SpanKind.INTERNAL, + ), + ( + RetrieverSpan( + name="knowledge-base", input="query", output=[Document(content="result")], created_at=CREATED_AT + ), + "retrieval knowledge-base", + SpanKind.INTERNAL, + ), + ( + WorkflowSpan(name="research", input="question", output="answer", created_at=CREATED_AT), + "invoke_workflow research", + SpanKind.INTERNAL, + ), + ( + AgentSpan(name="planner", input="request", output="plan", created_at=CREATED_AT), + "invoke_agent planner", + SpanKind.INTERNAL, + ), + (Trace(name="travel", input="request", output="plan", created_at=CREATED_AT), "travel", SpanKind.INTERNAL), + ( + ControlSpan( + name="guardrail", + input='{"text":"request"}', + output=ControlResult(action="observe", matched=True), + created_at=CREATED_AT, + ), + "guardrail", + SpanKind.INTERNAL, + ), + ] + + +@pytest.mark.parametrize(("span", "expected_name", "expected_kind"), supported_spans()) +def test_supported_span_types_use_canonical_names_and_kinds( + span: BaseStep, expected_name: str, expected_kind: SpanKind +) -> None: + result = convert(span) + + assert isinstance(result, ReadableSpan) + assert result.name == expected_name + assert result.kind is expected_kind + assert dict(result.attributes or {}) == build_span_attributes(span, "session-1") + + +@pytest.mark.parametrize( + ("span", "expected_name"), + [ + (LlmSpan(input="prompt", output="answer", model=None), "chat"), + (ToolSpan(name="", input="input"), "execute_tool"), + (RetrieverSpan(name="", input="query", output=[]), "retrieval"), + (WorkflowSpan(name="", input="input"), "invoke_workflow"), + (AgentSpan(name="", input="input"), "invoke_agent"), + ], +) +def test_missing_optional_name_parts_do_not_leave_whitespace(span: BaseStep, expected_name: str) -> None: + assert convert(span).name == expected_name + + +@pytest.mark.parametrize( + "span", [BaseStep(type=StepType.session), ToolSpan(name="tool").model_copy(update={"type": "unknown"})] +) +def test_unsupported_step_types_fail_clearly(span: BaseStep) -> None: + with pytest.raises(TypeError, match="Unsupported step type"): + convert(span) + + +def test_converter_calls_shared_attribute_builder_once(monkeypatch: pytest.MonkeyPatch) -> None: + source = ToolSpan(name="search", input="query") + expected = {"gen_ai.operation.name": "execute_tool", "custom": "value"} + calls: list[tuple[BaseStep, str | None]] = [] + + def build(source_span: BaseStep, session_id: str | None = None) -> dict[str, Any]: + calls.append((source_span, session_id)) + return expected + + monkeypatch.setattr(span_converter, "build_span_attributes", build) + + result = convert(source, session_id="conversation-1") + + assert calls == [(source, "conversation-1")] + assert result.attributes == expected + + +def test_converter_leaves_final_export_normalization_to_the_sink() -> None: + result = convert(LlmSpan(input="prompt", output="answer", model="gpt-5")) + attributes = result.attributes or {} + + assert "gen_ai.input.messages" in attributes + assert "gen_ai.output.messages" in attributes + assert "splunk_ao.input.messages" not in attributes + assert "splunk_ao.output.messages" not in attributes + assert "splunk_ao.system" not in attributes + + +def test_resource_routing_is_preserved_without_becoming_a_span_attribute() -> None: + resource = Resource.create( + {"service.name": "travel-agent", "splunk_ao.project.name": "project", "splunk_ao.logstream.name": "log-stream"} + ) + + result = convert(ToolSpan(name="search"), resource=resource) + + assert result.resource is resource + assert result.resource.attributes["splunk_ao.project.name"] == "project" + assert "splunk_ao.project.name" not in (result.attributes or {}) + assert "splunk_ao.logstream.name" not in (result.attributes or {}) + + +def test_context_parent_flags_and_trace_state_are_preserved() -> None: + trace_state = TraceState([("vendor", "state")]) + span_context = make_context(trace_flags=TraceFlags(0), trace_state=trace_state) + parent_context = make_context(span_id=PARENT_SPAN_ID, is_remote=True, trace_state=trace_state) + + result = convert( + LlmSpan(input="prompt", output="answer"), span_context=span_context, parent_span_context=parent_context + ) + + assert result.context is span_context + assert result.parent is parent_context + assert result.context.trace_id == parent_context.trace_id == TRACE_ID + assert result.context.span_id == SPAN_ID + assert result.parent.span_id == PARENT_SPAN_ID + assert result.context.trace_flags == TraceFlags(0) + assert result.context.trace_state is trace_state + assert result.parent.is_remote is True + + +def test_root_span_has_no_parent() -> None: + assert convert(Trace(name="root", input="question")).parent is None + + +def test_deep_parent_chain_uses_only_preassigned_contexts() -> None: + workflow_context = make_context(span_id=0x1111111111111111) + agent_context = make_context(span_id=0x2222222222222222) + llm_context = make_context(span_id=0x3333333333333333) + + workflow = convert(WorkflowSpan(name="workflow"), span_context=workflow_context) + agent = convert(AgentSpan(name="agent"), span_context=agent_context, parent_span_context=workflow_context) + llm = convert(LlmSpan(model="model"), span_context=llm_context, parent_span_context=agent_context) + + assert workflow.context is workflow_context + assert workflow.parent is None + assert agent.parent is workflow_context + assert llm.parent is agent_context + assert workflow.context.trace_id == agent.context.trace_id == llm.context.trace_id == TRACE_ID + + +@pytest.mark.parametrize( + ("created_at", "expected_start_ns"), + [ + (datetime(2025, 1, 1, 0, 0, 0, 123456, tzinfo=UTC), 1_735_689_600_123_456_000), + (datetime(2025, 1, 1, 0, 0, 0, 123456), 1_735_689_600_123_456_000), + ], +) +def test_created_at_converts_to_exact_unix_nanoseconds(created_at: datetime, expected_start_ns: int) -> None: + result = convert(ToolSpan(name="tool", created_at=created_at), end_time_ns=expected_start_ns + 1) + + assert result.start_time == expected_start_ns + + +def test_duration_takes_precedence_over_explicit_end_time_including_zero() -> None: + start_ns = 1_735_689_600_000_000_000 + with_duration = ToolSpan(name="tool", created_at=CREATED_AT, metrics=Metrics(duration_ns=25)) + with_zero_duration = ToolSpan(name="tool", created_at=CREATED_AT, metrics=Metrics(duration_ns=0)) + + assert convert(with_duration, end_time_ns=start_ns + 100).end_time == start_ns + 25 + assert convert(with_zero_duration, end_time_ns=start_ns + 100).end_time == start_ns + + +def test_explicit_end_time_precedes_wall_clock() -> None: + assert convert(ToolSpan(name="tool", created_at=CREATED_AT), end_time_ns=1234).end_time == 1234 + + +def test_end_time_falls_back_to_wall_clock(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(span_converter.time, "time_ns", lambda: 9876) + + assert convert(ToolSpan(name="tool", created_at=CREATED_AT)).end_time == 9876 + + +@pytest.mark.parametrize( + ("status_code", "expected"), + [ + (None, StatusCode.UNSET), + (200, StatusCode.UNSET), + (399, StatusCode.UNSET), + (400, StatusCode.ERROR), + (500, StatusCode.ERROR), + ], +) +def test_status_code_maps_to_otel_status(status_code: int | None, expected: StatusCode) -> None: + result = convert(ToolSpan(name="tool", status_code=status_code)) + + assert result.status.status_code is expected + assert result.status.description in (None, "") + + +class ToolSpanWithException(ToolSpan): + exception: str | None = None + + +def test_exception_sets_error_status_and_description() -> None: + result = convert(ToolSpanWithException(name="tool", exception="tool failed")) + + assert result.status.status_code is StatusCode.ERROR + assert result.status.description == "tool failed" + + +def test_readable_span_has_empty_events_links_and_versioned_scope() -> None: + result = convert(LlmSpan(input="prompt", output="answer", events=[{"type": "reasoning"}])) + + assert result.events == () + assert result.links == () + assert result.instrumentation_scope.name == "splunk_ao" + assert result.instrumentation_scope.version == get_package_version() + + +def test_dynamic_scorer_results_are_not_emitted() -> None: + source = LlmSpan(input="prompt", output="answer") + source.metrics.__dict__.update({"factuality": 0.9, "factuality_status": "passed"}) + + attributes = convert(source).attributes or {} + + assert "factuality" not in attributes + assert "factuality_status" not in attributes From cbc5c4df7d29aea4c143b83f2f2ceb9f98d4f58e Mon Sep 17 00:00:00 2001 From: pradystar Date: Fri, 24 Jul 2026 16:04:19 -0700 Subject: [PATCH 07/10] fix(otel): reject proprietary trace envelopes --- src/splunk_ao/converter/span_converter.py | 3 ++- tests/test_span_converter.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/splunk_ao/converter/span_converter.py b/src/splunk_ao/converter/span_converter.py index 2f4cefa7..85ec5d6b 100644 --- a/src/splunk_ao/converter/span_converter.py +++ b/src/splunk_ao/converter/span_converter.py @@ -23,7 +23,6 @@ StepType.retriever: ("retrieval", "name"), StepType.workflow: ("invoke_workflow", "name"), StepType.agent: ("invoke_agent", "name"), - StepType.trace: ("", "name"), StepType.control: ("", "name"), } @@ -39,6 +38,8 @@ def _step_type(span: BaseStep) -> StepType: except (TypeError, ValueError) as exc: raise TypeError(f"Unsupported step type: {raw_type}") from exc + if step_type is StepType.trace: + raise TypeError("LoggedTrace is a trace envelope and cannot be converted") if step_type not in _NAME_PARTS: raise TypeError(f"Unsupported step type: {step_type.value}") return step_type diff --git a/tests/test_span_converter.py b/tests/test_span_converter.py index ca983275..86733706 100644 --- a/tests/test_span_converter.py +++ b/tests/test_span_converter.py @@ -84,7 +84,6 @@ def supported_spans() -> list[tuple[BaseStep, str, SpanKind]]: "invoke_agent planner", SpanKind.INTERNAL, ), - (Trace(name="travel", input="request", output="plan", created_at=CREATED_AT), "travel", SpanKind.INTERNAL), ( ControlSpan( name="guardrail", @@ -132,6 +131,11 @@ def test_unsupported_step_types_fail_clearly(span: BaseStep) -> None: convert(span) +def test_trace_envelope_is_not_convertible() -> None: + with pytest.raises(TypeError, match="LoggedTrace is a trace envelope"): + convert(Trace(name="root", input="question")) + + def test_converter_calls_shared_attribute_builder_once(monkeypatch: pytest.MonkeyPatch) -> None: source = ToolSpan(name="search", input="query") expected = {"gen_ai.operation.name": "execute_tool", "custom": "value"} @@ -193,7 +197,7 @@ def test_context_parent_flags_and_trace_state_are_preserved() -> None: def test_root_span_has_no_parent() -> None: - assert convert(Trace(name="root", input="question")).parent is None + assert convert(LlmSpan(input="question")).parent is None def test_deep_parent_chain_uses_only_preassigned_contexts() -> None: From adf3cd594105a82bb4b795e1e9c314a3bdb56a14 Mon Sep 17 00:00:00 2001 From: pradystar Date: Fri, 24 Jul 2026 16:42:24 -0700 Subject: [PATCH 08/10] remove unsupported exception-based status mapping --- src/splunk_ao/converter/span_converter.py | 7 +++---- tests/test_span_converter.py | 12 ++++-------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/splunk_ao/converter/span_converter.py b/src/splunk_ao/converter/span_converter.py index 85ec5d6b..540dec0c 100644 --- a/src/splunk_ao/converter/span_converter.py +++ b/src/splunk_ao/converter/span_converter.py @@ -59,11 +59,10 @@ def _to_unix_ns(value: datetime) -> int: def _span_status(span: BaseStep) -> Status: - exception = getattr(span, "exception", None) - status_code = getattr(span, "status_code", None) - if exception is None and (status_code is None or status_code < 400): + status_code = span.status_code + if status_code is None or status_code < 400: return Status(StatusCode.UNSET) - return Status(StatusCode.ERROR, str(exception) if exception is not None else "") + return Status(StatusCode.ERROR) def _end_time_ns(span: BaseStep, start_time_ns: int, end_time_ns: int | None) -> int: diff --git a/tests/test_span_converter.py b/tests/test_span_converter.py index 86733706..ee67683e 100644 --- a/tests/test_span_converter.py +++ b/tests/test_span_converter.py @@ -262,18 +262,14 @@ def test_status_code_maps_to_otel_status(status_code: int | None, expected: Stat result = convert(ToolSpan(name="tool", status_code=status_code)) assert result.status.status_code is expected - assert result.status.description in (None, "") + assert result.status.description is None -class ToolSpanWithException(ToolSpan): - exception: str | None = None - - -def test_exception_sets_error_status_and_description() -> None: - result = convert(ToolSpanWithException(name="tool", exception="tool failed")) +def test_error_output_is_not_inferred_as_status_description() -> None: + result = convert(ToolSpan(name="tool", output="Error: tool failed", status_code=500)) assert result.status.status_code is StatusCode.ERROR - assert result.status.description == "tool failed" + assert result.status.description is None def test_readable_span_has_empty_events_links_and_versioned_scope() -> None: From bc51095da00135c59bc755140bdad1cb3aebb786 Mon Sep 17 00:00:00 2001 From: Pradeep Nair Date: Mon, 27 Jul 2026 14:41:40 -0700 Subject: [PATCH 09/10] defensive cast to prevent ValueError Co-authored-by: Fernando Correia --- src/splunk_ao/converter/attribute_mapping.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/splunk_ao/converter/attribute_mapping.py b/src/splunk_ao/converter/attribute_mapping.py index 7b09c6d6..af87a887 100644 --- a/src/splunk_ao/converter/attribute_mapping.py +++ b/src/splunk_ao/converter/attribute_mapping.py @@ -483,7 +483,10 @@ def build_span_attributes(span: BaseStep, session_id: str | None = None) -> dict def _alias_value(source_key: str, value: AttributeValue) -> AttributeValue: if source_key == "gen_ai.response.time_to_first_chunk": - return int(float(value) * 1_000_000_000) + try: + return int(float(value) * 1_000_000_000) + except (TypeError, ValueError): + return value return value From 5c2e392e86c170634e3638a1d4f72258e921360e Mon Sep 17 00:00:00 2001 From: pradystar Date: Mon, 27 Jul 2026 15:25:44 -0700 Subject: [PATCH 10/10] validate exporter configuration --- README.md | 5 +++++ .../src/splunk_ao_a2a/instrumentor.py | 4 ++-- src/splunk_ao/otel.py | 14 +++++++++++++- tests/test_otel_native_paths.py | 19 ++++++++++++++++++- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b26ca587..56180cea 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,11 @@ export SPLUNK_AO_PROJECT="your-project-name" export SPLUNK_AO_LOG_STREAM="your-log-stream-name" ``` +When using `SplunkAOSpanProcessor`, routing is captured when its exporter is +constructed and remains fixed for that exporter's lifetime. This matches the +OpenTelemetry Resource model and keeps request headers consistent for batched +spans. Use a separate processor and exporter for each additional destination. + Set `SPLUNK_AO_LOGGING_DISABLED=true` to disable telemetry collection and export. diff --git a/splunk-ao-a2a/src/splunk_ao_a2a/instrumentor.py b/splunk-ao-a2a/src/splunk_ao_a2a/instrumentor.py index 3db3bc62..eda808a6 100644 --- a/splunk-ao-a2a/src/splunk_ao_a2a/instrumentor.py +++ b/splunk-ao-a2a/src/splunk_ao_a2a/instrumentor.py @@ -32,11 +32,11 @@ class A2AInstrumentor(BaseInstrumentor): # type: ignore[misc] Example:: from opentelemetry.sdk.trace import TracerProvider - from splunk_ao.otel import SplunkAOSpanProcessor, add_splunk_ao_span_processor + from splunk_ao.otel import add_splunk_ao_span_processor from splunk_ao_a2a import A2AInstrumentor provider = TracerProvider() - add_splunk_ao_span_processor(provider, SplunkAOSpanProcessor()) + add_splunk_ao_span_processor(provider) A2AInstrumentor().instrument(tracer_provider=provider, agent_name="my-agent") # To disable message content capture (e.g. for PII compliance): diff --git a/src/splunk_ao/otel.py b/src/splunk_ao/otel.py index 65e4f0c6..b5349b98 100644 --- a/src/splunk_ao/otel.py +++ b/src/splunk_ao/otel.py @@ -148,7 +148,6 @@ def _with_routing_resource(span: ReadableSpan, routing_resource: Resource) -> Re events=span.events, links=span.links, kind=span.kind, - instrumentation_info=span.instrumentation_info, status=span.status, start_time=span.start_time, end_time=span.end_time, @@ -163,6 +162,10 @@ class SplunkAOOTLPExporter(SpanExporter): This exporter wraps the standard OTLPSpanExporter with deployment-aware configuration, authentication, and immutable routing. For most applications, use SplunkAOSpanProcessor instead, which provides a complete tracing solution. + + Routing is captured when the exporter is constructed and remains fixed for its + lifetime. Applications that export to multiple destinations must use a separate + exporter and span processor for each destination. """ def __init__( @@ -231,6 +234,8 @@ class SplunkAOSpanProcessor(SpanProcessor): This processor combines span processing and export capabilities into a single component that can be directly attached to any OpenTelemetry TracerProvider. It handles the complete lifecycle of spans from creation to export to Galileo. + Project, log-stream, and experiment routing is fixed when the processor's exporter + is constructed. Use separate processors and exporters for separate destinations. Examples -------- @@ -266,7 +271,14 @@ def __init__( SpanProcessor : type, optional Custom span processor class. Defaults to BatchSpanProcessor for optimal performance. + Raises + ------ + ValueError + When a prebuilt exporter is combined with exporter configuration options. """ + if _exporter is not None and kwargs: + raise ValueError("OTLP exporter options cannot be used with _exporter") + self._exporter = ( _exporter if _exporter is not None diff --git a/tests/test_otel_native_paths.py b/tests/test_otel_native_paths.py index 119b60e2..8d35c88c 100644 --- a/tests/test_otel_native_paths.py +++ b/tests/test_otel_native_paths.py @@ -1,3 +1,4 @@ +import warnings from collections.abc import Sequence from unittest.mock import MagicMock, patch @@ -286,7 +287,6 @@ def test_exporter_preserves_every_unaffected_span_field() -> None: "events", "links", "kind", - "instrumentation_info", "status", "start_time", "end_time", @@ -301,6 +301,18 @@ def test_exporter_preserves_every_unaffected_span_field() -> None: exporter.shutdown() +def test_exporter_does_not_read_deprecated_instrumentation_info() -> None: + factory = RecordingExporterFactory() + exporter = build_exporter(DeploymentMode.O11Y, factory, project="project") + + with warnings.catch_warnings(record=True) as captured: + warnings.simplefilter("always", DeprecationWarning) + exporter.export((make_span(),)) + + assert not any("instrumentation_scope" in str(warning.message) for warning in captured) + exporter.shutdown() + + def test_exporter_delegates_force_flush_and_shutdown() -> None: factory = RecordingExporterFactory() exporter = build_exporter(DeploymentMode.O11Y, factory) @@ -356,6 +368,11 @@ def test_processor_forwards_complete_routing_to_immutable_exporter() -> None: processor.shutdown() +def test_processor_rejects_prebuilt_exporter_with_otlp_options() -> None: + with pytest.raises(ValueError, match="OTLP exporter options cannot be used with _exporter"): + SplunkAOSpanProcessor(_exporter=RecordingExporter(), timeout=10) + + def test_helper_constructs_registers_and_returns_processor() -> None: provider = MagicMock() resolved_processor = MagicMock(spec=SplunkAOSpanProcessor)