Skip to content

feat(langchain): Record reasoning token usage attribute #7136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alexander-alderman-webb merged 18 commits into from
Aug 10, 2026
Merged

feat(langchain): Record reasoning token usage attribute #7136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
de5fc9d
fix(langchain): Stop double-counting usage in multi-candidate responses
alexander-alderman-webb Aug 6, 2026
026b909
toxfile
alexander-alderman-webb Aug 6, 2026
7fa4869
re-add test
alexander-alderman-webb Aug 6, 2026
c468eda
version gate
alexander-alderman-webb Aug 6, 2026
32ef8c5
config.py formatting
alexander-alderman-webb Aug 6, 2026
beb1080
redo tox
alexander-alderman-webb Aug 6, 2026
776ca00
feat(langchain): Record cached token usage values
alexander-alderman-webb Aug 7, 2026
f0e8114
clean up
alexander-alderman-webb Aug 7, 2026
2ef5347
make mypy happy
alexander-alderman-webb Aug 7, 2026
a88a3df
make mypy even happier
alexander-alderman-webb Aug 7, 2026
17498bb
mypy again
alexander-alderman-webb Aug 7, 2026
1d0bc01
.
alexander-alderman-webb Aug 7, 2026
dfafde1
restore test
alexander-alderman-webb Aug 7, 2026
b1c3907
.
alexander-alderman-webb Aug 7, 2026
de6ec8e
feat(langchain): Record reasoning token usage attribute
alexander-alderman-webb Aug 7, 2026
f59a152
.
alexander-alderman-webb Aug 7, 2026
e7aa616
set initial value to None
alexander-alderman-webb Aug 7, 2026
920ea2d
merge master
alexander-alderman-webb Aug 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions sentry_sdk/consts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -814,6 +814,12 @@ class SPANDATA:
Example: 100
"""

GEN_AI_USAGE_REASONING_OUTPUT_TOKENS = "gen_ai.usage.reasoning.output_tokens"
Comment thread
alexander-alderman-webb marked this conversation as resolved.
"""
The number of tokens used for reasoning to create the AI output.
Example: 75
"""

GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"
"""
The number of tokens in the output.
Expand Down
34 changes: 24 additions & 10 deletions sentry_sdk/integrations/langchain.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
total_tokens: "Optional[int]"
cache_read: "Optional[int]"
cache_creation: "Optional[int]"
reasoning: "Optional[int]"


try:
Expand Down Expand Up @@ -735,6 +736,7 @@
total_total = 0
total_cache_read = None
total_cache_creation = None
reasoning = None

for gen_list in generations:
if not gen_list:
Expand All @@ -761,25 +763,31 @@
continue

input_token_details = usage_metadata.get("input_token_details")
if not isinstance(input_token_details, dict):
continue
if isinstance(input_token_details, dict):
if isinstance(input_token_details.get("cache_read"), int):
total_cache_read = (total_cache_read or 0) + input_token_details[
"cache_read"
]

if isinstance(input_token_details.get("cache_read"), int):
total_cache_read = (total_cache_read or 0) + input_token_details[
"cache_read"
]
if isinstance(input_token_details.get("cache_creation"), int):
total_cache_creation = (
total_cache_creation or 0
) + input_token_details["cache_creation"]

if isinstance(input_token_details.get("cache_creation"), int):
total_cache_creation = (total_cache_creation or 0) + input_token_details[
"cache_creation"
]
output_token_details = usage_metadata.get("output_token_details")
if not isinstance(output_token_details, dict):
continue

if isinstance(output_token_details.get("reasoning"), int):
reasoning = (reasoning or 0) + output_token_details["reasoning"]

return TokenUsage(
total_input if total_input > 0 else None,
total_output if total_output > 0 else None,
total_total if total_total > 0 else None,
total_cache_read,
total_cache_creation,
reasoning,
Comment thread
alexander-alderman-webb marked this conversation as resolved.
)


Expand Down Expand Up @@ -819,6 +827,7 @@
total_tokens = None
cache_read_tokens = None
cache_creation_tokens = None
reasoning = None

# Legacy that reads provider-specific token information.
token_usage = _get_token_usage(response)
Expand All @@ -838,6 +847,8 @@
cache_read_tokens = token_usage.cache_read
if token_usage.cache_creation is not None:
cache_creation_tokens = token_usage.cache_creation
if token_usage.reasoning is not None:
reasoning = token_usage.reasoning

set_on_span = (
span.set_attribute if isinstance(span, StreamedSpan) else span.set_data
Expand All @@ -860,6 +871,9 @@
SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, cache_creation_tokens
)

if reasoning is not None:
set_on_span(SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, reasoning)

Check warning on line 875 in sentry_sdk/integrations/langchain.py

View check run for this annotation

@sentry/warden / warden: find-bugs

LangChain uses duplicate reasoning token attribute, fragmenting data across integrations 

Use the existing `SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING` attribute instead of introducing a duplicate `SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS` constant.
Comment thread
alexander-alderman-webb marked this conversation as resolved.


def _get_request_data(
obj: "Any", args: "Any", kwargs: "Any"
Expand Down
43 changes: 39 additions & 4 deletions tests/integrations/langchain/test_langchain.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ def nonstreaming_multi_candidate_google_genai_model_response():
cached_content_token_count=4,
prompt_token_count=10,
candidates_token_count=20,
thoughts_token_count=5,
total_token_count=30,
),
)
Expand Down Expand Up @@ -591,7 +592,7 @@ def test_langchain_multi_choice_response(
assert len(chat_spans) == 1

assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 10
assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 20
assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 25
Comment thread
alexander-alderman-webb marked this conversation as resolved.
assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30

assert (
Expand All @@ -616,7 +617,7 @@ def test_langchain_multi_choice_response(
assert len(chat_spans) == 1

assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 10
assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 20
assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 25
assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 30

assert chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 4
Expand Down Expand Up @@ -817,6 +818,11 @@ def test_langchain_create_agent(
== 6
)

assert (
chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]
== 5
)

if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13):
assert (
chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4"
Expand Down Expand Up @@ -904,6 +910,11 @@ def test_langchain_create_agent(
== 6
)

assert (
chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]
== 5
)

if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13):
assert (
chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4"
Expand Down Expand Up @@ -979,6 +990,8 @@ def test_langchain_create_agent(
== 6
)

assert chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 5

if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13):
assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4"

Expand Down Expand Up @@ -1054,7 +1067,7 @@ def test_tool_execution_span(
),
output_tokens=50,
output_tokens_details=OutputTokensDetails(
reasoning_tokens=0,
reasoning_tokens=10,
),
total_tokens=192,
),
Expand All @@ -1066,7 +1079,7 @@ def test_tool_execution_span(
),
output_tokens=28,
output_tokens_details=OutputTokensDetails(
reasoning_tokens=0,
reasoning_tokens=11,
),
total_tokens=117,
),
Expand Down Expand Up @@ -1153,6 +1166,10 @@ def test_tool_execution_span(
]
== 31
)
assert (
chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]
== 10
)
assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat"

assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89
Expand All @@ -1168,6 +1185,10 @@ def test_tool_execution_span(
]
== 10
)
assert (
chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]
== 11
)
assert chat_spans[1]["attributes"]["gen_ai.system"] == "openai-chat"

if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13):
Expand Down Expand Up @@ -1286,6 +1307,10 @@ def test_tool_execution_span(
]
== 31
)
assert (
chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]
== 10
)
assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat"

assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89
Expand All @@ -1301,6 +1326,10 @@ def test_tool_execution_span(
]
== 10
)
assert (
chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]
== 11
)
assert chat_spans[1]["attributes"]["gen_ai.system"] == "openai-chat"

if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13):
Expand Down Expand Up @@ -1416,6 +1445,9 @@ def test_tool_execution_span(
chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]
== 31
)
assert (
chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 10
)
assert chat_spans[0]["data"]["gen_ai.system"] == "openai-chat"

assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89
Expand All @@ -1428,6 +1460,9 @@ def test_tool_execution_span(
chat_spans[1]["data"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]
== 10
)
assert (
chat_spans[1]["data"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 11
)
assert chat_spans[1]["data"]["gen_ai.system"] == "openai-chat"

if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13):
Expand Down
Loading