Closed
Description
Summary of problem
Async requests with vision returns a 431 from openai
I have confirmed this only happens when using datadog as when I replace this
ENTRYPOINT ["ddtrace-run"]
CMD ["uvicorn", "app.main:app" , "--host", "0.0.0.0", "--port", "80"]
with
ENTRYPOINT []
CMD ["uvicorn", "app.main:app" , "--host", "0.0.0.0", "--port", "80"]
It doesn't happen.
When I run the get_descriptions
from a fastapi route
#!/usr/bin/env python
import asyncio
import base64
import io
import time
from typing import Literal, TypedDict
from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate
from langchain.schema import SystemMessage
from langchain_community.callbacks import get_openai_callback
from langchain_core.output_parsers import StrOutputParser
from langchain_openai import AzureChatOpenAI
from PIL import Image
ImageFormat = Literal["JPEG", "PNG"]
ImageURL = TypedDict("ImageURL", {"url": str, "detail": str})
ImageContentObj = TypedDict("ImageContentObj", {"type": str, "image_url": ImageURL})
def _convert_image_to_base64_string(image: Image.Image, fmt: ImageFormat = "JPEG") -> str:
"""Convert image to a base64 utf-8 encoded string"""
bytes_io = io.BytesIO()
image.save(bytes_io, format=fmt)
img_str = base64.b64encode(bytes_io.getvalue()).decode("utf-8")
return img_str
def _create_image(width: int, height: int, color: tuple[int, int, int]):
image = Image.new("RGB", (width, height), color)
return image
async def get_descriptions(llm: AzureChatOpenAI):
# llm = get_llm(model_type="4o-mini")
image0 = _create_image(50, 50, (255, 0, 0))
image1 = _create_image(50, 50, (0, 255, 0))
image2 = _create_image(50, 50, (0, 0, 255))
system_message = SystemMessage("Descripe this image")
human_message_prompt_template = HumanMessagePromptTemplate.from_template(
template=[
{ # pyright: ignore[reportArgumentType]
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,{image_base64}", "detail": "low"},
}
]
)
prompt_template = ChatPromptTemplate(
input_variables=["image_base64"],
messages=[system_message, human_message_prompt_template],
)
chain = prompt_template | llm | StrOutputParser()
t1 = time.time()
with get_openai_callback() as cb:
descriptions = await asyncio.gather(
*(
chain.ainvoke({"image_base64": _convert_image_to_base64_string(image)})
for image in (image0, image1, image2)
)
)
took = time.time() - t1
for description in descriptions:
print(description)
print(
f"Took {took:,.2f}s to get descriptions for 3 images, "
f"token consumption (Total | Prompt | Completion): {cb.total_tokens} | "
f"{cb.prompt_tokens} | {cb.completion_tokens}, cost: ${cb.total_cost:.5f}"
)
if __name__ == "__main__":
llm = AzureChatOpenAI()
asyncio.run(get_descriptions(llm))
I get
descriptions = await asyncio.gather(
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/ddtrace/contrib/langchain/patch.py", line 879, in traced_lcel_runnable_sequence_async
final_output = await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/langchain_core/runnables/base.py", line 2920, in ainvoke
input = await asyncio.create_task(part(), context=context) # type: ignore
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py", line 298, in ainvoke
llm_result = await self.agenerate_prompt(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py", line 787, in agenerate_prompt
return await self.agenerate(
^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/ddtrace/contrib/langchain/patch.py", line 523, in traced_chat_model_agenerate
chat_completions = await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py", line 747, in agenerate
raise exceptions[0]
File "/usr/local/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py", line 923, in _agenerate_with_cache
result = await self._agenerate(
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/langchain_openai/chat_models/base.py", line 752, in _agenerate
response = await self.async_client.create(**payload)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/ddtrace/contrib/openai/patch.py", line 290, in patched_endpoint
resp = await func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/openai/resources/chat/completions.py", line 1295, in create
return await self._post(
^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/openai/_base_client.py", line 1826, in post
return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/openai/_base_client.py", line 1519, in request
return await self._request(
^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/site-packages/openai/_base_client.py", line 1620, in _request
raise self._make_status_error_from_response(err.response) from None
openai.APIStatusError: Error code: 431
This doesn't seem to happen with the non azure chat, i.e. ChatOpenAI
doesn't seem to have this issue
Which version of dd-trace-py are you using?
ddtrace==2.11.1
Which version of pip are you using?
Not using pip, using uv
uv --version
uv 0.3.4 (Homebrew 2024-08-26)
Which libraries and their versions are you using?
aiobotocore==2.13.2
aiohappyeyeballs==2.3.4
aiohttp==3.10.3
aioitertools==0.11.0
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
astroid==3.2.4
attrs==23.2.0
azure-common==1.1.28
azure-core==1.30.2
azure-identity==1.17.1
azure-search==1.0.0b2
azure-search-documents==11.5.1
basedpyright==1.17.0
beautifulsoup4==4.12.3
blinker==1.8.2
boto3==1.34.131
boto3-stubs==1.34.160
botocore==1.34.131
botocore-stubs==1.34.160
bytecode==0.15.1
cattrs==23.2.3
certifi==2024.7.4
cffi==1.16.0
cfgv==3.4.0
charset-normalizer==3.3.2
click==8.1.7
cryptography==43.0.0
dataclasses-json==0.6.7
ddsketch==3.0.1
ddtrace==2.11.1
debugpy==1.8.2
deprecated==1.2.14
dill==0.3.8
distlib==0.3.8
distro==1.9.0
dnspython==2.6.1
docker==7.1.0
docopt==0.6.2
dynaconf==3.2.6
email-validator==2.2.0
envier==0.5.2
et-xmlfile==1.1.0
fastapi==0.111.1
fastapi-cli==0.0.4
filelock==3.15.4
flask==3.0.0
freezegun==1.5.1
frozenlist==1.4.1
git-python==1.0.3
gitdb==4.0.11
gitpython==3.1.43
greenlet==3.0.3
h11==0.14.0
hiredis==2.4.0
httpcore==1.0.5
httptools==0.6.1
httpx==0.27.0
identify==2.6.0
idna==3.7
importlib-metadata==8.0.0
iniconfig==2.0.0
isodate==0.6.1
isort==5.13.2
itsdangerous==2.2.0
jinja2==3.1.4
jmespath==1.0.1
jsonpatch==1.33
jsonpointer==3.0.0
langchain==0.2.15
langchain-community==0.2.10
langchain-core==0.2.36
langchain-openai==0.1.20
langchain-text-splitters==0.2.2
langsmith==0.1.95
language-tags==1.2.0
lxml==5.3.0
markdown-it-py==3.0.0
markupsafe==2.1.5
marshmallow==3.21.3
mccabe==0.7.0
mdurl==0.1.2
more-itertools==10.3.0
moto==5.0.11
msal==1.30.0
msal-extensions==1.2.0
msrest==0.7.1
multidict==6.0.5
mypy-boto3-s3==1.34.158
mypy-extensions==1.0.0
nodeenv==1.9.1
nodejs-wheel-binaries==20.16.0
numpy==1.26.4
oauthlib==3.2.2
openai==1.37.1
openpyxl==3.1.5
opentelemetry-api==1.26.0
orjson==3.10.6
packaging==24.1
pandas==2.2.2
pikepdf==9.1.2
pillow==10.4.0
platformdirs==4.2.2
playwright==1.45.1
pluggy==1.5.0
portalocker==2.10.1
pre-commit==3.8.0
protobuf==5.27.3
pycparser==2.22
pycryptodome==3.20.0
pydantic==2.4.2
pydantic-core==2.10.1
pyee==11.1.0
pygments==2.18.0
pyjwt==2.8.0
pylint==3.2.6
pylint-protobuf==0.22.0
pymupdf==1.24.9
pymupdfb==1.24.9
pynamodb==6.0.1
pynamodb-attributes==0.5.0
pytest==8.3.2
pytest-asyncio==0.23.8
pytest-mock==3.14.0
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
python-json-logger==2.0.7
python-multipart==0.0.9
python-ulid==1.1.0
pytz==2024.1
pyyaml==6.0.1
redis==5.0.8
redis-om==0.2.2
regex==2024.7.24
requests==2.32.3
requests-mock==1.12.1
requests-oauthlib==2.0.0
requests-toolbelt==1.0.0
responses==0.25.3
rich==13.7.1
s3transfer==0.10.2
setuptools==69.5.1
shellingham==1.5.4
six==1.16.0
smmap==5.0.1
sniffio==1.3.1
soupsieve==2.5
sqlalchemy==2.0.31
sse-starlette==2.1.3
sseclient-py==1.8.0
starlette==0.37.2
stomp-py==8.1.2
tenacity==8.5.0
termcolor==2.4.0
tiktoken==0.7.0
tinydb==4.8.0
tomlkit==0.13.0
tqdm==4.66.4
typer==0.12.3
types-aiobotocore==2.13.2.post1
types-aiobotocore-lambda==2.13.2
types-aiobotocore-s3==2.13.3
types-awscrt==0.21.2
types-cffi==1.16.0.20240331
types-pyopenssl==24.1.0.20240722
types-redis==4.6.0.20240726
types-s3transfer==0.10.1
types-setuptools==71.1.0.20240726
typing-extensions==4.12.2
typing-inspect==0.9.0
tzdata==2024.1
urllib3==2.2.2
uvicorn==0.30.4
uvloop==0.19.0
virtualenv==20.26.3
watchfiles==0.22.0
websocket-client==1.8.0
websockets==12.0
werkzeug==3.0.3
wrapt==1.16.0
xmltodict==0.13.0
yarl==1.9.4
zipp==3.19.2
How can we reproduce your problem?
Create a fastapi route and run the above
What is the result that you get?
Error is above
What is the result that you expected?
No error