setup v0 and v1 tests side-by-side (Azure#32671)

* setup openai tests against v0 and v1

* fixes to get tests working side-by-sie

* fix global replace mistake

* more tests for different ways to build the client + ad token auth

* reorganize client tests

* try testing against v1 PoC in CI

* update dev reqs

* fix attr access for azure in audio

* adjust tests to pass for audio/images (for now)

* tests for cli and models

* testing module client

* few fixes to get things working in CI + avoid flaky test where text is empty string

* adapt tests to latest changes

* get get_bearer_token_provider from azure-identity release

* adapt to changes in latest commit

* adapt tests to latest commit

* point to v1 branch now that azure client merged

Author: kristapratico

sdk/openai/azure-openai/dev_requirements.txt

@@ -1,6 +1,6 @@
 -e ../../../tools/azure-sdk-tools
 -e ../../../tools/azure-devtools
 ../../identity/azure-identity
-../../core/azure-core
 numpy
-git+https://github.com/openai/openai-python@main
+aiohttp
+requests

sdk/openai/azure-openai/setup.py

@@ -62,9 +62,7 @@
         'azure.openai': ['py.typed'],
     },
     install_requires=[
-        "azure-core<2.0.0,>=1.26.0",
-        "openai<1.0.0,>=0.27.6",
-        "azure-identity<2.0.0,>=1.12.0"
+        "azure-identity<2.0.0,>=1.15.0"
     ],
     python_requires=">=3.7",
     project_urls={

sdk/openai/azure-openai/tests/conftest.py

@@ -6,23 +6,34 @@
 # --------------------------------------------------------------------------
 import os
 import pytest
+import importlib
+import contextlib
 import requests
 import aiohttp
 import yarl
 import functools
 import openai
 from devtools_testutils.sanitizers import add_header_regex_sanitizer, add_oauth_response_sanitizer
-from azure.identity import DefaultAzureCredential
+from azure.identity import DefaultAzureCredential, get_bearer_token_provider
+from azure.identity.aio import (
+    DefaultAzureCredential as AsyncDefaultAzureCredential,
+    get_bearer_token_provider as get_bearer_token_provider_async,
+)
 
+# controls whether we run tests against v0 or v1. Options: v0 or v1. Default: v1
+ENV_OPENAI_TEST_MODE = "OPENAI_TEST_MODE"
 
 # for pytest.parametrize
 ALL = ["azure", "azuread", "openai"]
 AZURE = "azure"
 OPENAI = "openai"
 AZURE_AD = "azuread"
+WHISPER_AZURE = "whisper_azure"
+WHISPER_AZURE_AD = "whisper_azuread"
+WHISPER_ALL = ["whisper_azure", "whisper_azuread", "openai"]
 
 # Environment variable keys
-ENV_AZURE_OPENAI_ENDPOINT = "AZURE_OPENAI_ENDPOINT"
+ENV_AZURE_OPENAI_ENDPOINT = "AZ_OPENAI_ENDPOINT"
 ENV_AZURE_OPENAI_KEY = "AZURE_OPENAI_KEY"
 ENV_AZURE_OPENAI_WHISPER_ENDPOINT = "AZURE_OPENAI_WHISPER_ENDPOINT"
 ENV_AZURE_OPENAI_WHISPER_KEY = "AZURE_OPENAI_WHISPER_KEY"
@@ -88,22 +99,162 @@ def azure_openai_creds():
         "audio_model": ENV_OPENAI_AUDIO_MODEL,
     }
 
+# openai>=1.0.0 ---------------------------------------------------------------------------
 
-def configure_api_type(api_type, whisper=False, **kwargs):
+@pytest.fixture
+def client(api_type):
+    if os.getenv(ENV_OPENAI_TEST_MODE, "v1") != "v1":
+        pytest.skip("Skipping - tests set to run against v1.")
     if api_type == "azure":
-        if whisper:
-            openai.api_base = os.getenv(ENV_AZURE_OPENAI_WHISPER_ENDPOINT).rstrip("/")
-            openai.api_key = os.getenv(ENV_AZURE_OPENAI_WHISPER_KEY)
-        else:
-            openai.api_base = os.getenv(ENV_AZURE_OPENAI_ENDPOINT).rstrip("/")
-            openai.api_key = os.getenv(ENV_AZURE_OPENAI_KEY)
+        client = openai.AzureOpenAI(
+            azure_endpoint=os.getenv(ENV_AZURE_OPENAI_ENDPOINT),
+            api_key=os.getenv(ENV_AZURE_OPENAI_KEY),
+            api_version=ENV_AZURE_OPENAI_API_VERSION,
+        )
+    elif api_type == "azuread":
+        client = openai.AzureOpenAI(
+            azure_endpoint=os.getenv(ENV_AZURE_OPENAI_ENDPOINT),
+            azure_ad_token_provider=get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"),
+            api_version=ENV_AZURE_OPENAI_API_VERSION,
+        )
+    elif api_type == "openai":
+        client = openai.OpenAI(
+            api_key=os.getenv(ENV_OPENAI_KEY)
+        )
+    elif api_type == "whisper_azure":
+        client = openai.AzureOpenAI(
+            azure_endpoint=os.getenv(ENV_AZURE_OPENAI_WHISPER_ENDPOINT),
+            api_key=os.getenv(ENV_AZURE_OPENAI_WHISPER_KEY),
+            api_version=ENV_AZURE_OPENAI_API_VERSION,
+        )
+    elif api_type == "whisper_azuread":
+        client = openai.AzureOpenAI(
+            azure_endpoint=os.getenv(ENV_AZURE_OPENAI_WHISPER_ENDPOINT),
+            azure_ad_token_provider=get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"),
+            api_version=ENV_AZURE_OPENAI_API_VERSION,
+        )
+
+    return client
+
+
+@pytest.fixture
+def client_async(api_type):
+    if os.getenv(ENV_OPENAI_TEST_MODE, "v1") != "v1":
+        pytest.skip("Skipping - tests set to run against v1.")
+    if api_type == "azure":
+        client = openai.AsyncAzureOpenAI(
+            azure_endpoint=os.getenv(ENV_AZURE_OPENAI_ENDPOINT),
+            api_key=os.getenv(ENV_AZURE_OPENAI_KEY),
+            api_version=ENV_AZURE_OPENAI_API_VERSION,
+        )
+    elif api_type == "azuread":
+        client = openai.AsyncAzureOpenAI(
+            azure_endpoint=os.getenv(ENV_AZURE_OPENAI_ENDPOINT),
+            azure_ad_token_provider=get_bearer_token_provider_async(AsyncDefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"),
+            api_version=ENV_AZURE_OPENAI_API_VERSION,
+        )
+    elif api_type == "openai":
+        client = openai.AsyncOpenAI(
+            api_key=os.getenv(ENV_OPENAI_KEY)
+        )
+    elif api_type == "whisper_azure":
+        client = openai.AsyncAzureOpenAI(
+            azure_endpoint=os.getenv(ENV_AZURE_OPENAI_WHISPER_ENDPOINT),
+            api_key=os.getenv(ENV_AZURE_OPENAI_WHISPER_KEY),
+            api_version=ENV_AZURE_OPENAI_API_VERSION,
+        )
+    elif api_type == "whisper_azuread":
+        client = openai.AsyncAzureOpenAI(
+            azure_endpoint=os.getenv(ENV_AZURE_OPENAI_WHISPER_ENDPOINT),
+            azure_ad_token_provider=get_bearer_token_provider_async(AsyncDefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"),
+            api_version=ENV_AZURE_OPENAI_API_VERSION,
+        )
+
+    return client
+
+
+def build_kwargs(args, api_type):
+    test_feature = args[0].qualified_test_name
+    if test_feature.startswith("test_audio"):
+        if api_type in ["whisper_azure", "whisper_azuread"]:
+            return {"model": ENV_AZURE_OPENAI_AUDIO_NAME}
+        elif api_type == "openai":
+            return {"model": ENV_OPENAI_AUDIO_MODEL}
+    if test_feature.startswith("test_chat_completions") \
+        or test_feature.startswith(("test_client", "test_models")):
+        if api_type in ["azure", "azuread"]:
+            return {"model": ENV_AZURE_OPENAI_CHAT_COMPLETIONS_NAME}
+        elif api_type == "openai":
+            return {"model": ENV_OPENAI_CHAT_COMPLETIONS_MODEL}
+    if test_feature.startswith("test_completions"):
+        if api_type in ["azure", "azuread"]:
+            return {"model": ENV_AZURE_OPENAI_COMPLETIONS_NAME}
+        elif api_type == "openai":
+            return {"model": ENV_OPENAI_COMPLETIONS_MODEL}
+    if test_feature.startswith("test_embeddings"):
+        if api_type in ["azure", "azuread"]:
+            return {"model": ENV_AZURE_OPENAI_EMBEDDINGS_NAME}
+        elif api_type == "openai":
+            return {"model": ENV_OPENAI_EMBEDDINGS_MODEL}
+    if test_feature.startswith(("test_dall_e", "test_module_client", "test_cli")):
+        return {}
+    raise ValueError(f"Test feature: {test_feature} needs to have its kwargs configured.")
+
+
+def configure_async(f):
+    @functools.wraps(f)
+    async def wrapper(*args, **kwargs):
+        api_type = kwargs.pop("api_type")
+        client_async = kwargs.pop("client_async")
+        azure_openai_creds = kwargs.pop("azure_openai_creds")
+        kwargs = build_kwargs(args, api_type)
+        try:
+            return await f(*args, client_async=client_async, azure_openai_creds=azure_openai_creds, api_type=api_type, **kwargs)
+        except openai.RateLimitError:
+            pytest.skip(f"{str(f).split(' ')[1]}[{api_type}]: Skipping - Rate limit reached.")
+
+    return wrapper
+
+
+def configure(f):
+    @functools.wraps(f)
+    def wrapper(*args, **kwargs):
+        api_type = kwargs.pop("api_type")
+        client = kwargs.pop("client")
+        azure_openai_creds = kwargs.pop("azure_openai_creds")
+        kwargs = build_kwargs(args, api_type)
+        try:
+            return f(*args, client=client, azure_openai_creds=azure_openai_creds, api_type=api_type, **kwargs)
+        except openai.RateLimitError:
+            pytest.skip(f"{str(f).split(' ')[1]}[{api_type}]: Skipping - Rate limit reached.")
+
+    return wrapper
+
+
+
+@contextlib.contextmanager
+def reload():
+    try:
+        importlib.reload(openai)
+        yield
+    finally:
+        importlib.reload(openai)
+
+
+# openai<1.0.0 ---------------------------------------------------------------------------
+
+@pytest.fixture
+def set_vars(api_type):
+    if os.getenv(ENV_OPENAI_TEST_MODE, "v1") != "v0":
+        pytest.skip("Skipping - tests set to run against v0.")
+
+    if api_type == "azure":
+        openai.api_base = os.getenv(ENV_AZURE_OPENAI_ENDPOINT).rstrip("/")
+        openai.api_key = os.getenv(ENV_AZURE_OPENAI_KEY)
         openai.api_type = "azure"
         openai.api_version = ENV_AZURE_OPENAI_API_VERSION
     elif api_type == "azuread":
-        if whisper:
-            openai.api_base = os.getenv(ENV_AZURE_OPENAI_WHISPER_ENDPOINT).rstrip("/")
-        else:
-            openai.api_base = os.getenv(ENV_AZURE_OPENAI_ENDPOINT).rstrip("/")
+        openai.api_base = os.getenv(ENV_AZURE_OPENAI_ENDPOINT).rstrip("/")
         credential = DefaultAzureCredential()
         token = credential.get_token("https://cognitiveservices.azure.com/.default")
         openai.api_type = "azuread"
@@ -114,30 +265,42 @@ def configure_api_type(api_type, whisper=False, **kwargs):
         openai.api_type = "openai"
         openai.api_key = os.getenv(ENV_OPENAI_KEY)
         openai.api_version = None
+    elif api_type == "whisper_azure":
+        openai.api_base = os.getenv(ENV_AZURE_OPENAI_WHISPER_ENDPOINT).rstrip("/")
+        openai.api_key = os.getenv(ENV_AZURE_OPENAI_WHISPER_KEY)
+        openai.api_type = "azure"
+        openai.api_version = ENV_AZURE_OPENAI_API_VERSION
+    elif api_type == "whisper_azuread":
+        openai.api_base = os.getenv(ENV_AZURE_OPENAI_WHISPER_ENDPOINT).rstrip("/")
+        credential = DefaultAzureCredential()
+        token = credential.get_token("https://cognitiveservices.azure.com/.default")
+        openai.api_type = "azuread"
+        openai.api_key = token.token
+        openai.api_version = ENV_AZURE_OPENAI_API_VERSION
 
 
-def configure_async(f):
+def configure_v0_async(f):
     @functools.wraps(f)
     async def wrapper(*args, **kwargs):
         api_type = kwargs.pop("api_type")
-        whisper = args[0].qualified_test_name.startswith("test_audio")
-        configure_api_type(api_type, whisper=whisper, **kwargs)
+        set_vars = kwargs.pop("set_vars")
+        azure_openai_creds = kwargs.pop("azure_openai_creds")
         try:
-            return await f(*args, api_type=api_type, **kwargs)
+            return await f(*args, set_vars=set_vars, azure_openai_creds=azure_openai_creds, api_type=api_type, **kwargs)
         except openai.error.RateLimitError:
             pytest.skip(f"{str(f).split(' ')[1]}[{api_type}]: Skipping - Rate limit reached.")
 
     return wrapper
 
 
-def configure(f):
+def configure_v0(f):
     @functools.wraps(f)
     def wrapper(*args, **kwargs):
         api_type = kwargs.pop("api_type")
-        whisper = args[0].qualified_test_name.startswith("test_audio")
-        configure_api_type(api_type, whisper=whisper, **kwargs)
+        set_vars = kwargs.pop("set_vars")
+        azure_openai_creds = kwargs.pop("azure_openai_creds")
         try:
-            return f(*args, api_type=api_type, **kwargs)
+            return f(*args, set_vars=set_vars, azure_openai_creds=azure_openai_creds, api_type=api_type, **kwargs)
         except openai.error.RateLimitError:
             pytest.skip(f"{str(f).split(' ')[1]}[{api_type}]: Skipping - Rate limit reached.")