Update tests and instrumentation for gpt-5.1.

newrelic/hooks/mlmodel_openai.py

@@ -303,7 +303,7 @@ def _record_embedding_success(transaction, embedding_id, linking_metadata, kwarg
             "duration": ft.duration * 1000,
             "response.model": response_model,
             "response.organization": organization,
-            "response.headers.llmVersion": response_headers.get("openai-version"),
+            "response.headers.llmVersion": response_headers.get("openai-version") or None,
             "response.headers.ratelimitLimitRequests": check_rate_limit_header(
                 response_headers, "x-ratelimit-limit-requests", True
             ),
@@ -459,7 +459,7 @@ def _handle_completion_success(
             return_val._nr_openai_attrs = getattr(return_val, "_nr_openai_attrs", {})
             return_val._nr_openai_attrs["messages"] = kwargs.get("messages", [])
             return_val._nr_openai_attrs["temperature"] = kwargs.get("temperature")
-            return_val._nr_openai_attrs["max_tokens"] = kwargs.get("max_tokens")
+            return_val._nr_openai_attrs["max_tokens"] = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens")
             return_val._nr_openai_attrs["model"] = kwargs.get("model") or kwargs.get("engine")
             return
         except Exception:
@@ -532,7 +532,8 @@ def _record_completion_success(
             "trace_id": trace_id,
             "request.model": request_model,
             "request.temperature": kwargs.get("temperature"),
-            "request.max_tokens": kwargs.get("max_tokens"),
+            # Later gpt models may use "max_completion_tokens" instead of "max_tokens"
+            "request.max_tokens": kwargs.get("max_tokens") or kwargs.get("max_completion_tokens"),
             "vendor": "openai",
             "ingest_source": "Python",
             "request_id": request_id,
@@ -648,7 +649,7 @@ def _record_completion_error(transaction, linking_metadata, completion_id, kwarg
             "response.number_of_messages": len(request_message_list),
             "request.model": request_model,
             "request.temperature": kwargs.get("temperature"),
-            "request.max_tokens": kwargs.get("max_tokens"),
+            "request.max_tokens": kwargs.get("max_tokens") or kwargs.get("max_completion_tokens"),
             "vendor": "openai",
             "ingest_source": "Python",
             "response.organization": exc_organization,

tests/mlmodel_openai/test_chat_completion_error_v1.py

@@ -115,7 +115,7 @@ def test_chat_completion_invalid_request_error_no_model(set_trace_info, sync_ope
         add_custom_attribute("llm.conversation_id", "my-awesome-id")
         with WithLlmCustomAttributes({"context": "attr"}):
             sync_openai_client.chat.completions.create(
-                messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100
+                messages=_test_openai_chat_completion_messages, temperature=0.7, max_completion_tokens=100
             )
 
 
@@ -141,7 +141,7 @@ def test_chat_completion_invalid_request_error_no_model_no_content(set_trace_inf
     with pytest.raises(TypeError):
         add_custom_attribute("llm.conversation_id", "my-awesome-id")
         sync_openai_client.chat.completions.create(
-            messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100
+            messages=_test_openai_chat_completion_messages, temperature=0.7, max_completion_tokens=100
         )
 
 
@@ -170,7 +170,7 @@ def test_chat_completion_invalid_request_error_no_model_async(loop, set_trace_in
         with WithLlmCustomAttributes({"context": "attr"}):
             loop.run_until_complete(
                 async_openai_client.chat.completions.create(
-                    messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100
+                    messages=_test_openai_chat_completion_messages, temperature=0.7, max_completion_tokens=100
                 )
             )
 
@@ -198,7 +198,7 @@ def test_chat_completion_invalid_request_error_no_model_async_no_content(loop, s
         add_custom_attribute("llm.conversation_id", "my-awesome-id")
         loop.run_until_complete(
             async_openai_client.chat.completions.create(
-                messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100
+                messages=_test_openai_chat_completion_messages, temperature=0.7, max_completion_tokens=100
             )
         )
 
@@ -267,7 +267,7 @@ def test_chat_completion_invalid_request_error_invalid_model(set_trace_info, syn
             model="does-not-exist",
             messages=({"role": "user", "content": "Model does not exist."},),
             temperature=0.7,
-            max_tokens=100,
+            max_completion_tokens=100,
         )
 
 
@@ -298,7 +298,7 @@ def test_chat_completion_invalid_request_error_invalid_model_with_token_count(se
             model="does-not-exist",
             messages=({"role": "user", "content": "Model does not exist."},),
             temperature=0.7,
-            max_tokens=100,
+            max_completion_tokens=100,
         )
 
 
@@ -329,7 +329,7 @@ def test_chat_completion_invalid_request_error_invalid_model_async(loop, set_tra
                 model="does-not-exist",
                 messages=({"role": "user", "content": "Model does not exist."},),
                 temperature=0.7,
-                max_tokens=100,
+                max_completion_tokens=100,
             )
         )
 
@@ -364,7 +364,7 @@ def test_chat_completion_invalid_request_error_invalid_model_with_token_count_as
                 model="does-not-exist",
                 messages=({"role": "user", "content": "Model does not exist."},),
                 temperature=0.7,
-                max_tokens=100,
+                max_completion_tokens=100,
             )
         )
 
@@ -378,7 +378,7 @@ def test_chat_completion_invalid_request_error_invalid_model_with_token_count_as
             "span_id": None,
             "trace_id": "trace-id",
             "duration": None,  # Response time varies each test run
-            "request.model": "gpt-3.5-turbo",
+            "request.model": "gpt-5.1",
             "request.temperature": 0.7,
             "request.max_tokens": 100,
             "response.number_of_messages": 1,
@@ -430,10 +430,10 @@ def test_chat_completion_wrong_api_key_error(monkeypatch, set_trace_info, sync_o
     monkeypatch.setattr(sync_openai_client, "api_key", "DEADBEEF")
     with pytest.raises(openai.AuthenticationError):
         sync_openai_client.chat.completions.create(
-            model="gpt-3.5-turbo",
+            model="gpt-5.1",
             messages=({"role": "user", "content": "Invalid API key."},),
             temperature=0.7,
-            max_tokens=100,
+            max_completion_tokens=100,
         )
 
 
@@ -463,10 +463,10 @@ def test_chat_completion_wrong_api_key_error_async(loop, monkeypatch, set_trace_
     with pytest.raises(openai.AuthenticationError):
         loop.run_until_complete(
             async_openai_client.chat.completions.create(
-                model="gpt-3.5-turbo",
+                model="gpt-5.1",
                 messages=({"role": "user", "content": "Invalid API key."},),
                 temperature=0.7,
-                max_tokens=100,
+                max_completion_tokens=100,
             )
         )
 
@@ -493,7 +493,7 @@ def test_chat_completion_invalid_request_error_no_model_with_raw_response(set_tr
     with pytest.raises(TypeError):
         add_custom_attribute("llm.conversation_id", "my-awesome-id")
         sync_openai_client.chat.completions.with_raw_response.create(
-            messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100
+            messages=_test_openai_chat_completion_messages, temperature=0.7, max_completion_tokens=100
         )
 
 
@@ -522,7 +522,7 @@ def test_chat_completion_invalid_request_error_no_model_no_content_with_raw_resp
     with pytest.raises(TypeError):
         add_custom_attribute("llm.conversation_id", "my-awesome-id")
         sync_openai_client.chat.completions.with_raw_response.create(
-            messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100
+            messages=_test_openai_chat_completion_messages, temperature=0.7, max_completion_tokens=100
         )
 
 
@@ -551,7 +551,7 @@ def test_chat_completion_invalid_request_error_no_model_async_with_raw_response(
         add_custom_attribute("llm.conversation_id", "my-awesome-id")
         loop.run_until_complete(
             async_openai_client.chat.completions.with_raw_response.create(
-                messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100
+                messages=_test_openai_chat_completion_messages, temperature=0.7, max_completion_tokens=100
             )
         )
 
@@ -582,7 +582,7 @@ def test_chat_completion_invalid_request_error_no_model_async_no_content_with_ra
         add_custom_attribute("llm.conversation_id", "my-awesome-id")
         loop.run_until_complete(
             async_openai_client.chat.completions.with_raw_response.create(
-                messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100
+                messages=_test_openai_chat_completion_messages, temperature=0.7, max_completion_tokens=100
             )
         )
 
@@ -613,7 +613,7 @@ def test_chat_completion_invalid_request_error_invalid_model_with_raw_response(s
             model="does-not-exist",
             messages=({"role": "user", "content": "Model does not exist."},),
             temperature=0.7,
-            max_tokens=100,
+            max_completion_tokens=100,
         )
 
 
@@ -646,7 +646,7 @@ def test_chat_completion_invalid_request_error_invalid_model_with_token_count_wi
             model="does-not-exist",
             messages=({"role": "user", "content": "Model does not exist."},),
             temperature=0.7,
-            max_tokens=100,
+            max_completion_tokens=100,
         )
 
 
@@ -679,7 +679,7 @@ def test_chat_completion_invalid_request_error_invalid_model_async_with_raw_resp
                 model="does-not-exist",
                 messages=({"role": "user", "content": "Model does not exist."},),
                 temperature=0.7,
-                max_tokens=100,
+                max_completion_tokens=100,
             )
         )
 
@@ -714,7 +714,7 @@ def test_chat_completion_invalid_request_error_invalid_model_with_token_count_as
                 model="does-not-exist",
                 messages=({"role": "user", "content": "Model does not exist."},),
                 temperature=0.7,
-                max_tokens=100,
+                max_completion_tokens=100,
             )
         )
 
@@ -744,10 +744,10 @@ def test_chat_completion_wrong_api_key_error_with_raw_response(monkeypatch, set_
     monkeypatch.setattr(sync_openai_client, "api_key", "DEADBEEF")
     with pytest.raises(openai.AuthenticationError):
         sync_openai_client.chat.completions.with_raw_response.create(
-            model="gpt-3.5-turbo",
+            model="gpt-5.1",
             messages=({"role": "user", "content": "Invalid API key."},),
             temperature=0.7,
-            max_tokens=100,
+            max_completion_tokens=100,
         )
 
 
@@ -779,9 +779,9 @@ def test_chat_completion_wrong_api_key_error_async_with_raw_response(
     with pytest.raises(openai.AuthenticationError):
         loop.run_until_complete(
             async_openai_client.chat.completions.with_raw_response.create(
-                model="gpt-3.5-turbo",
+                model="gpt-5.1",
                 messages=({"role": "user", "content": "Invalid API key."},),
                 temperature=0.7,
-                max_tokens=100,
+                max_completion_tokens=100,
             )
         )

tests/mlmodel_openai/test_chat_completion_stream_error_v1.py

@@ -116,7 +116,7 @@ def test_chat_completion_invalid_request_error_no_model(set_trace_info, sync_ope
         add_custom_attribute("llm.conversation_id", "my-awesome-id")
         with WithLlmCustomAttributes({"context": "attr"}):
             generator = sync_openai_client.chat.completions.create(
-                messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100, stream=True
+                messages=_test_openai_chat_completion_messages, temperature=0.7, max_completion_tokens=100, stream=True
             )
             for resp in generator:
                 assert resp
@@ -145,7 +145,7 @@ def test_chat_completion_invalid_request_error_no_model_no_content(set_trace_inf
     with pytest.raises(TypeError):
         add_custom_attribute("llm.conversation_id", "my-awesome-id")
         generator = sync_openai_client.chat.completions.create(
-            messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100, stream=True
+            messages=_test_openai_chat_completion_messages, temperature=0.7, max_completion_tokens=100, stream=True
         )
         for resp in generator:
             assert resp
@@ -176,7 +176,10 @@ def test_chat_completion_invalid_request_error_no_model_async(loop, set_trace_in
 
             async def consumer():
                 generator = await async_openai_client.chat.completions.create(
-                    messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100, stream=True
+                    messages=_test_openai_chat_completion_messages,
+                    temperature=0.7,
+                    max_completion_tokens=100,
+                    stream=True,
                 )
                 async for resp in generator:
                     assert resp
@@ -209,7 +212,7 @@ def test_chat_completion_invalid_request_error_no_model_async_no_content(loop, s
 
         async def consumer():
             generator = await async_openai_client.chat.completions.create(
-                messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100, stream=True
+                messages=_test_openai_chat_completion_messages, temperature=0.7, max_completion_tokens=100, stream=True
             )
             async for resp in generator:
                 assert resp
@@ -261,7 +264,9 @@ async def consumer():
     callable_name(openai.NotFoundError),
     exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "model_not_found", "http.statusCode": 404}},
 )
-@validate_span_events(exact_agents={"error.message": "The model `does-not-exist` does not exist"})
+@validate_span_events(
+    exact_agents={"error.message": "The model `does-not-exist` does not exist or you do not have access to it."}
+)
 @validate_transaction_metrics(
     "test_chat_completion_stream_error_v1:test_chat_completion_invalid_request_error_invalid_model",
     scoped_metrics=[("Llm/completion/OpenAI/create", 1)],
@@ -279,7 +284,7 @@ def test_chat_completion_invalid_request_error_invalid_model(set_trace_info, syn
             model="does-not-exist",
             messages=({"role": "user", "content": "Model does not exist."},),
             temperature=0.7,
-            max_tokens=100,
+            max_completion_tokens=100,
             stream=True,
         )
         for resp in generator:
@@ -293,7 +298,9 @@ def test_chat_completion_invalid_request_error_invalid_model(set_trace_info, syn
     callable_name(openai.NotFoundError),
     exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "model_not_found", "http.statusCode": 404}},
 )
-@validate_span_events(exact_agents={"error.message": "The model `does-not-exist` does not exist"})
+@validate_span_events(
+    exact_agents={"error.message": "The model `does-not-exist` does not exist or you do not have access to it."}
+)
 @validate_transaction_metrics(
     "test_chat_completion_stream_error_v1:test_chat_completion_invalid_request_error_invalid_model_with_token_count",
     scoped_metrics=[("Llm/completion/OpenAI/create", 1)],
@@ -312,7 +319,7 @@ def test_chat_completion_invalid_request_error_invalid_model_with_token_count(se
             model="does-not-exist",
             messages=({"role": "user", "content": "Model does not exist."},),
             temperature=0.7,
-            max_tokens=100,
+            max_completion_tokens=100,
             stream=True,
         )
         for resp in generator:
@@ -326,7 +333,9 @@ def test_chat_completion_invalid_request_error_invalid_model_with_token_count(se
     callable_name(openai.NotFoundError),
     exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "model_not_found", "http.statusCode": 404}},
 )
-@validate_span_events(exact_agents={"error.message": "The model `does-not-exist` does not exist"})
+@validate_span_events(
+    exact_agents={"error.message": "The model `does-not-exist` does not exist or you do not have access to it."}
+)
 @validate_transaction_metrics(
     "test_chat_completion_stream_error_v1:test_chat_completion_invalid_request_error_invalid_model_async_with_token_count",
     scoped_metrics=[("Llm/completion/OpenAI/create", 1)],
@@ -348,7 +357,7 @@ async def consumer():
                 model="does-not-exist",
                 messages=({"role": "user", "content": "Model does not exist."},),
                 temperature=0.7,
-                max_tokens=100,
+                max_completion_tokens=100,
                 stream=True,
             )
             async for resp in generator:
@@ -363,7 +372,9 @@ async def consumer():
     callable_name(openai.NotFoundError),
     exact_attrs={"agent": {}, "intrinsic": {}, "user": {"error.code": "model_not_found", "http.statusCode": 404}},
 )
-@validate_span_events(exact_agents={"error.message": "The model `does-not-exist` does not exist"})
+@validate_span_events(
+    exact_agents={"error.message": "The model `does-not-exist` does not exist or you do not have access to it."}
+)
 @validate_transaction_metrics(
     "test_chat_completion_stream_error_v1:test_chat_completion_invalid_request_error_invalid_model_async",
     scoped_metrics=[("Llm/completion/OpenAI/create", 1)],
@@ -383,7 +394,7 @@ async def consumer():
                 model="does-not-exist",
                 messages=({"role": "user", "content": "Model does not exist."},),
                 temperature=0.7,
-                max_tokens=100,
+                max_completion_tokens=100,
                 stream=True,
             )
             async for resp in generator:
@@ -401,7 +412,7 @@ async def consumer():
             "span_id": None,
             "trace_id": "trace-id",
             "duration": None,  # Response time varies each test run
-            "request.model": "gpt-3.5-turbo",
+            "request.model": "gpt-5.1",
             "request.temperature": 0.7,
             "request.max_tokens": 100,
             "response.number_of_messages": 1,
@@ -453,10 +464,10 @@ def test_chat_completion_wrong_api_key_error(monkeypatch, set_trace_info, sync_o
     monkeypatch.setattr(sync_openai_client, "api_key", "DEADBEEF")
     with pytest.raises(openai.AuthenticationError):
         generator = sync_openai_client.chat.completions.create(
-            model="gpt-3.5-turbo",
+            model="gpt-5.1",
             messages=({"role": "user", "content": "Invalid API key."},),
             temperature=0.7,
-            max_tokens=100,
+            max_completion_tokens=100,
             stream=True,
         )
         for resp in generator:
@@ -490,10 +501,10 @@ def test_chat_completion_wrong_api_key_error_async(loop, monkeypatch, set_trace_
 
         async def consumer():
             generator = await async_openai_client.chat.completions.create(
-                model="gpt-3.5-turbo",
+                model="gpt-5.1",
                 messages=({"role": "user", "content": "Invalid API key."},),
                 temperature=0.7,
-                max_tokens=100,
+                max_completion_tokens=100,
                 stream=True,
             )
             async for resp in generator: