I use a callback to calculate tokens for an openai model. When trying it with cohere, I noticed that response.llm_output is None. Is this normal?
"""
Custom openai callback
https://python.langchain.com/docs/concepts/callbacks/
"""
from datetime import datetime, timezone
from typing import Any, Dict, List
from uuid import UUID
from langchain_core.callbacks import BaseCallbackHandler
from langchain_core.messages import BaseMessage
from langchain_core.outputs import LLMResult
from pydantic import BaseModel
from app.usage import get_service_id
from .get_service_id import get_service_id
from .task import LLMTask, LLMTokenUsage, Task
class TokenUsage(BaseModel):
"""Token usage"""
input_tokens: int
output_tokens: int
total_tokens: int
class LLMOutput(BaseModel):
"""
Output included in llm response
"""
token_usage: TokenUsage
model: str
# system_fingerprint: str
class EmbeddingStartInfo(BaseModel):
start_date_time: datetime
total_tokens: int
class CustomOpenAICallback(BaseCallbackHandler):
"""Callback to intercept token information"""
def __init__(self):
self.tasks: list[Task] = []
self.run_start_times: dict[UUID, datetime] = {}
"""Dictionnary of run_id -> start_time to be able to execute multiple requests in parallel and get the correct timings"""
self.run_embedding_start_info: dict[UUID, EmbeddingStartInfo] = {}
def on_chat_model_start(
self,
serialized: Dict[str, Any],
messages: List[List[BaseMessage]],
*,
run_id: UUID,
parent_run_id: UUID | None = None,
tags: List[str] | None = None,
metadata: Dict[str, Any] | None = None,
**kwargs: Any,
):
"""When a chat model starts."""
self.run_start_times[run_id] = datetime.now(tz=timezone.utc)
def on_llm_start(
self,
serialized: Dict[str, Any],
prompts: List[str],
*,
run_id: UUID,
parent_run_id: UUID | None = None,
tags: List[str] | None = None,
metadata: Dict[str, Any] | None = None,
**kwargs: Any,
):
"""When a llm starts"""
# TODO do we use completion models? Should the logic be the same?
def on_llm_end(self, response: LLMResult, *, run_id: UUID, parent_run_id: UUID | None = None, **kwargs: Any):
"""
When an llm OR chat model ends.
Use information returned by the OpenAI API response.
"""
if response.llm_output is None:
return
end_date_time = datetime.now(tz=timezone.utc)
start_date_time = self.run_start_times.get(run_id)
if start_date_time is None:
raise RuntimeError(f"Start time not found for run_id: {run_id}")
output = LLMOutput(**response.llm_output)
task = LLMTask(
service_id=get_service_id(model_name=output.model),
model_name=output.model,
token_usage=LLMTokenUsage(
input_tokens=output.token_usage.input_tokens,
output_tokens=output.token_usage.output_tokens,
),
start_date_time=start_date_time,
end_date_time=end_date_time,
)
self.tasks.append(task)Here is what the response object looks like:
LLMResult(
generations=[
[
ChatGeneration(
text='...',
generation_info={
'id': '2a87fc55-57e6-4065-9afa-a283265b1699',
'finish_reason': 'COMPLETE',
'content': '...',
'token_count': {'input_tokens': 5037.0, 'output_tokens': 1189.0}
},
message=AIMessage(
content='...',
additional_kwargs={
'id': '2a87fc55-57e6-4065-9afa-a283265b1699',
'finish_reason': 'COMPLETE',
'content': '...',
'token_count': {'input_tokens': 5037.0, 'output_tokens': 1189.0}
},
response_metadata={
'id': '2a87fc55-57e6-4065-9afa-a283265b1699',
'finish_reason': 'COMPLETE',
'content': '...',
'token_count': {'input_tokens': 5037.0, 'output_tokens': 1189.0}
},
id='run--6fa795d7-75e1-4e84-b9bb-07a4c554e6af-0',
usage_metadata={'input_tokens': 5037, 'output_tokens': 1189, 'total_tokens': 6226}
)
)
]
],
llm_output=None,
run=None,
type='LLMResult'
)I am using version 0.4.6
root@f313e273197e:/app# poetry debug resolve --tree --install langchain-cohere
Resolving dependencies... (22.9s)
Resolution results:
langchain-cohere 0.4.6 An integration package connecting Cohere and LangChain
├── cohere >=5.18.0,<6.0
│ ├── fastavro >=1.9.4,<2.0.0
│ ├── httpx >=0.21.2
│ │ ├── anyio *
│ │ │ ├── exceptiongroup >=1.0.2
│ │ │ │ └── typing-extensions >=4.6.0
│ │ │ ├── idna >=2.8
│ │ │ ├── sniffio >=1.1
│ │ │ └── typing-extensions >=4.5 (circular dependency aborted here)
│ │ ├── certifi *
│ │ ├── httpcore ==1.*
│ │ │ ├── certifi * (circular dependency aborted here)
│ │ │ └── h11 >=0.16
│ │ └── idna * (circular dependency aborted here)
│ ├── httpx-sse 0.4.0
│ ├── pydantic >=1.9.2
│ │ ├── annotated-types >=0.6.0
│ │ ├── pydantic-core 2.33.2
│ │ │ └── typing-extensions >=4.6.0,<4.7.0 || >4.7.0 (circular dependency aborted here)
│ │ ├── typing-extensions >=4.12.2 (circular dependency aborted here)
│ │ └── typing-inspection >=0.4.0
│ │ └── typing-extensions >=4.12.0 (circular dependency aborted here)
│ ├── pydantic-core >=2.18.2,<3.0.0 (circular dependency aborted here)
│ ├── requests >=2.0.0,<3.0.0
│ │ ├── certifi >=2017.4.17 (circular dependency aborted here)
│ │ ├── charset-normalizer >=2,<4
│ │ ├── idna >=2.5,<4 (circular dependency aborted here)
│ │ └── urllib3 >=1.21.1,<3
│ ├── tokenizers >=0.15,<1
│ │ └── huggingface-hub >=0.16.4,<2.0
│ │ ├── filelock *
│ │ ├── fsspec >=2023.5.0
│ │ ├── hf-xet >=1.1.3,<2.0.0
│ │ ├── packaging >=20.9
│ │ ├── pyyaml >=5.1
│ │ ├── requests * (circular dependency aborted here)
│ │ ├── tqdm >=4.42.1
│ │ │ └── colorama *
│ │ └── typing-extensions >=3.7.4.3 (circular dependency aborted here)
│ ├── types-requests >=2.0.0,<3.0.0
│ │ └── urllib3 >=2 (circular dependency aborted here)
│ └── typing-extensions >=4.0.0 (circular dependency aborted here)
├── langchain-community >=0.3.0,<0.4.0
│ ├── aiohttp >=3.8.3,<4.0.0
│ │ ├── aiohappyeyeballs >=2.5.0
│ │ ├── aiosignal >=1.4.0
│ │ │ ├── frozenlist >=1.1.0
│ │ │ └── typing-extensions >=4.2
│ │ ├── async-timeout >=4.0,<6.0
│ │ ├── attrs >=17.3.0
│ │ ├── frozenlist >=1.1.1 (circular dependency aborted here)
│ │ ├── multidict >=4.5,<7.0
│ │ │ └── typing-extensions >=4.1.0 (circular dependency aborted here)
│ │ ├── propcache >=0.2.0
│ │ └── yarl >=1.17.0,<2.0
│ │ ├── idna >=2.0
│ │ ├── multidict >=4.0 (circular dependency aborted here)
│ │ └── propcache >=0.2.1 (circular dependency aborted here)
│ ├── dataclasses-json >=0.6.7,<0.7
│ │ ├── marshmallow >=3.18.0,<4.0.0
│ │ │ └── packaging >=17.0
│ │ └── typing-inspect >=0.4.0,<1
│ │ ├── mypy-extensions >=0.3.0
│ │ └── typing-extensions >=3.7.4 (circular dependency aborted here)
│ ├── httpx-sse >=0.4.0,<1.0.0
│ ├── langchain >=0.3.27,<2.0.0
│ │ ├── async-timeout >=4.0.0,<5.0.0 (circular dependency aborted here)
│ │ ├── langchain-core >=0.3.72,<1.0.0
│ │ │ ├── jsonpatch >=1.33,<2.0
│ │ │ │ └── jsonpointer >=1.9
│ │ │ ├── langsmith >=0.3.45
│ │ │ │ ├── httpx >=0.23.0,<1
│ │ │ │ │ ├── anyio *
│ │ │ │ │ │ ├── exceptiongroup >=1.0.2
│ │ │ │ │ │ │ └── typing-extensions >=4.6.0 (circular dependency aborted here)
│ │ │ │ │ │ ├── idna >=2.8 (circular dependency aborted here)
│ │ │ │ │ │ ├── sniffio >=1.1
│ │ │ │ │ │ └── typing-extensions >=4.5 (circular dependency aborted here)
│ │ │ │ │ ├── certifi *
│ │ │ │ │ ├── httpcore ==1.*
│ │ │ │ │ │ ├── certifi * (circular dependency aborted here)
│ │ │ │ │ │ └── h11 >=0.16
│ │ │ │ │ └── idna * (circular dependency aborted here)
│ │ │ │ ├── orjson >=3.9.14
│ │ │ │ ├── packaging >=23.2 (circular dependency aborted here)
│ │ │ │ ├── pydantic >=1,<3
│ │ │ │ │ ├── annotated-types >=0.6.0
│ │ │ │ │ ├── pydantic-core 2.33.2
│ │ │ │ │ │ └── typing-extensions >=4.6.0,<4.7.0 || >4.7.0 (circular dependency aborted here)
│ │ │ │ │ ├── typing-extensions >=4.12.2 (circular dependency aborted here)
│ │ │ │ │ └── typing-inspection >=0.4.0
│ │ │ │ │ └── typing-extensions >=4.12.0 (circular dependency aborted here)
│ │ │ │ ├── requests >=2.0.0
│ │ │ │ │ ├── certifi >=2017.4.17 (circular dependency aborted here)
│ │ │ │ │ ├── charset-normalizer >=2,<4
│ │ │ │ │ ├── idna >=2.5,<4 (circular dependency aborted here)
│ │ │ │ │ └── urllib3 >=1.21.1,<3
│ │ │ │ ├── requests-toolbelt >=1.0.0
│ │ │ │ │ └── requests >=2.0.1,<3.0.0 (circular dependency aborted here)
│ │ │ │ └── zstandard >=0.23.0
│ │ │ ├── packaging >=23.2 (circular dependency aborted here)
│ │ │ ├── pydantic >=2.7.4 (circular dependency aborted here)
│ │ │ ├── pyyaml >=5.3
│ │ │ ├── tenacity >=8.1.0,<8.4.0 || >8.4.0,<10.0.0
│ │ │ └── typing-extensions >=4.7 (circular dependency aborted here)
│ │ ├── langchain-text-splitters >=0.3.9,<1.0.0
│ │ │ └── langchain-core >=0.3.75,<2.0.0 (circular dependency aborted here)
│ │ ├── langsmith >=0.1.17 (circular dependency aborted here)
│ │ ├── pydantic >=2.7.4,<3.0.0 (circular dependency aborted here)
│ │ ├── pyyaml >=5.3 (circular dependency aborted here)
│ │ ├── requests >=2,<3 (circular dependency aborted here)
│ │ └── sqlalchemy >=1.4,<3
│ │ ├── greenlet >=1
│ │ └── typing-extensions >=4.6.0 (circular dependency aborted here)
│ ├── langchain-core >=0.3.75,<2.0.0 (circular dependency aborted here)
│ ├── langsmith >=0.1.125 (circular dependency aborted here)
│ ├── numpy >=1.26.2
│ ├── numpy >=2.1.0 (circular dependency aborted here)
│ ├── pydantic-settings >=2.10.1,<3.0.0
│ │ ├── pydantic >=2.7.0 (circular dependency aborted here)
│ │ ├── python-dotenv >=0.21.0
│ │ └── typing-inspection >=0.4.0 (circular dependency aborted here)
│ ├── pyyaml >=5.3 (circular dependency aborted here)
│ ├── requests >=2.32.5,<3 (circular dependency aborted here)
│ ├── sqlalchemy >=1.4,<3 (circular dependency aborted here)
│ └── tenacity >=8.1.0,<8.4.0 || >8.4.0,<10 (circular dependency aborted here)
├── langchain-core >=0.3.76,<0.4.0
│ ├── jsonpatch >=1.33,<2.0
│ │ └── jsonpointer >=1.9
│ ├── langsmith >=0.3.45
│ │ ├── httpx >=0.23.0,<1
│ │ │ ├── anyio *
│ │ │ │ ├── exceptiongroup >=1.0.2
│ │ │ │ │ └── typing-extensions >=4.6.0
│ │ │ │ ├── idna >=2.8
│ │ │ │ ├── sniffio >=1.1
│ │ │ │ └── typing-extensions >=4.5 (circular dependency aborted here)
│ │ │ ├── certifi *
│ │ │ ├── httpcore ==1.*
│ │ │ │ ├── certifi * (circular dependency aborted here)
│ │ │ │ └── h11 >=0.16
│ │ │ └── idna * (circular dependency aborted here)
│ │ ├── orjson >=3.9.14
│ │ ├── packaging >=23.2
│ │ ├── pydantic >=1,<3
│ │ │ ├── annotated-types >=0.6.0
│ │ │ ├── pydantic-core 2.33.2
│ │ │ │ └── typing-extensions >=4.6.0,<4.7.0 || >4.7.0 (circular dependency aborted here)
│ │ │ ├── typing-extensions >=4.12.2 (circular dependency aborted here)
│ │ │ └── typing-inspection >=0.4.0
│ │ │ └── typing-extensions >=4.12.0 (circular dependency aborted here)
│ │ ├── requests >=2.0.0
│ │ │ ├── certifi >=2017.4.17 (circular dependency aborted here)
│ │ │ ├── charset-normalizer >=2,<4
│ │ │ ├── idna >=2.5,<4 (circular dependency aborted here)
│ │ │ └── urllib3 >=1.21.1,<3
│ │ ├── requests-toolbelt >=1.0.0
│ │ │ └── requests >=2.0.1,<3.0.0 (circular dependency aborted here)
│ │ └── zstandard >=0.23.0
│ ├── packaging >=23.2 (circular dependency aborted here)
│ ├── pydantic >=2.7.4 (circular dependency aborted here)
│ ├── pyyaml >=5.3
│ ├── tenacity >=8.1.0,<8.4.0 || >8.4.0,<10.0.0
│ └── typing-extensions >=4.7 (circular dependency aborted here)
├── pydantic >=2,<3
│ ├── annotated-types >=0.6.0
│ ├── pydantic-core 2.33.2
│ │ └── typing-extensions >=4.6.0,<4.7.0 || >4.7.0
│ ├── typing-extensions >=4.12.2 (circular dependency aborted here)
│ └── typing-inspection >=0.4.0
│ └── typing-extensions >=4.12.0 (circular dependency aborted here)
└── types-pyyaml >=6.0.12.20240917,<7.0.0.0
I use a callback to calculate tokens for an openai model. When trying it with cohere, I noticed that
response.llm_outputisNone. Is this normal?Here is what the response object looks like:
I am using version 0.4.6