[FEATURE] Add a great-expectations user-agent suffix to S3 clients

docs/docusaurus/docs/core/connect_to_data/filesystem_data/_create_a_data_source/_s3/_s3.md

@@ -43,7 +43,7 @@ import PrereqDataContext from '../../../../_core_components/prerequisites/_preco
    The parameter `boto3_options` allows you to pass the following information:
 
    - `region_name`: Your AWS region name.
-   - `endpoint_url`: specifies an S3 endpoint.  You can provide an environment variable reference such as `"${S3_ENDPOINT}"` to securely include this in your code.  The string `"${S3_ENDPOINT}"` will be replaced with the value of the environment variable `S3_ENDPOINT`.
+   - `endpoint_url`: specifies an S3 endpoint. Set this to connect to a non-AWS S3-compatible object store, such as Backblaze B2, Cloudflare R2, or MinIO; leave it unset for Amazon S3.  You can provide an environment variable reference such as `"${S3_ENDPOINT}"` to securely include this in your code.  The string `"${S3_ENDPOINT}"` will be replaced with the value of the environment variable `S3_ENDPOINT`.
 
    For more information on secure storage and retrieval of credentials in GX see [Configure credentials](/core/connect_to_data/sql_data/sql_data.md#configure-credentials).
 
@@ -98,4 +98,3 @@ import PrereqDataContext from '../../../../_core_components/prerequisites/_preco
 </TabItem>
 
 </Tabs>
-

great_expectations/compatibility/aws.py

@@ -1,5 +1,8 @@
 from __future__ import annotations
 
+from importlib import metadata
+from typing import Any, Dict
+
 from great_expectations.compatibility.not_imported import NotImported
 
 BOTO_NOT_IMPORTED = NotImported(
@@ -32,6 +35,43 @@
 except ImportError:
     exceptions = BOTO_NOT_IMPORTED
 
+
+def _get_distribution_version() -> str:
+    try:
+        return metadata.version("great_expectations")
+    except metadata.PackageNotFoundError:
+        return "dev"
+
+
+def get_s3_boto3_options(boto3_options: Dict[str, Any]) -> Dict[str, Any]:
+    """Return boto3 client options with a ``great-expectations`` agent suffix.
+
+    Works with Amazon S3 and any S3-compatible object store (for example
+    Backblaze B2, Cloudflare R2, or MinIO). A caller-supplied ``config`` and
+    ``endpoint_url`` are preserved; the suffix is appended to an existing agent
+    string rather than replacing it.
+    """
+    suffix = f"great-expectations/{_get_distribution_version()}"
+    options = dict(boto3_options)
+
+    if isinstance(Config, NotImported):
+        return options
+
+    config = options.get("config")
+    if config is not None and not isinstance(config, Config):
+        return options
+
+    existing = getattr(config, "user_agent_extra", None) if config is not None else None
+    user_agent_extra = f"{existing} {suffix}" if existing else suffix
+
+    if config is not None:
+        options["config"] = config.merge(Config(user_agent_extra=user_agent_extra))
+    else:
+        options["config"] = Config(user_agent_extra=user_agent_extra)
+
+    return options
+
+
 try:
     import sqlalchemy_redshift
 except ImportError:

great_expectations/datasource/fluent/pandas_s3_datasource.py

@@ -68,7 +68,10 @@ def _get_s3_client(self) -> BaseClient:
                     "boto3_options", {}
                 )
                 try:
-                    s3_client = aws.boto3.client("s3", **boto3_options)
+                    s3_client = aws.boto3.client(
+                        "s3",
+                        **aws.get_s3_boto3_options(boto3_options),
+                    )
                 except Exception as e:
                     # Failure to create "s3_client" is most likely due invalid "boto3_options" dictionary.  # noqa: E501 # FIXME CoP
                     raise PandasS3DatasourceError(  # noqa: TRY003 # FIXME CoP

great_expectations/datasource/fluent/spark_s3_datasource.py

@@ -70,7 +70,10 @@ def _get_s3_client(self) -> BaseClient:
                     "boto3_options", {}
                 )
                 try:
-                    s3_client = aws.boto3.client("s3", **boto3_options)
+                    s3_client = aws.boto3.client(
+                        "s3",
+                        **aws.get_s3_boto3_options(boto3_options),
+                    )
                 except Exception as e:
                     # Failure to create "s3_client" is most likely due invalid "boto3_options" dictionary.  # noqa: E501 # FIXME CoP
                     raise SparkS3DatasourceError(  # noqa: TRY003 # FIXME CoP

great_expectations/execution_engine/pandas_execution_engine.py

@@ -167,7 +167,8 @@ def _instantiate_azure_client(self) -> None:
     def _instantiate_s3_client(self) -> None:
         # If s3_client was passed in (from data source) use it, otherwise create our own
         self._s3 = self._config.get("s3_client") or aws.boto3.client(
-            "s3", **self.config.get("boto3_options", {})
+            "s3",
+            **aws.get_s3_boto3_options(self.config.get("boto3_options", {})),
         )
 
     def _instantiate_gcs_client(self) -> None:

tests/compatibility/test_aws.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from typing import Any, Dict
+
+import pytest
+
+from great_expectations.compatibility import aws
+
+botocore_client = pytest.importorskip("botocore.client")
+
+pytestmark = pytest.mark.unit
+
+
+@pytest.fixture
+def distribution_version(monkeypatch: pytest.MonkeyPatch) -> str:
+    version = "1.2.3"
+    monkeypatch.setattr(aws, "_get_distribution_version", lambda: version)
+    return version
+
+
+def test_get_s3_boto3_options_adds_user_agent_suffix(distribution_version: str) -> None:
+    options = aws.get_s3_boto3_options({})
+
+    assert options["config"].user_agent_extra == f"great-expectations/{distribution_version}"
+
+
+def test_get_s3_boto3_options_appends_user_agent_and_preserves_options(
+    distribution_version: str,
+) -> None:
+    config = botocore_client.Config(user_agent_extra="my-app/1.0")
+    boto3_options: Dict[str, Any] = {
+        "config": config,
+        "endpoint_url": "https://s3.example.com",
+    }
+
+    options = aws.get_s3_boto3_options(boto3_options)
+
+    assert (
+        options["config"].user_agent_extra
+        == f"my-app/1.0 great-expectations/{distribution_version}"
+    )
+    assert options["endpoint_url"] == "https://s3.example.com"
+    assert boto3_options["config"] is config
+
+
+def test_get_s3_boto3_options_preserves_invalid_config() -> None:
+    boto3_options: Dict[str, Any] = {
+        "config": {"user_agent_extra": "invalid"},
+        "endpoint_url": "https://s3.example.com",
+    }
+
+    options = aws.get_s3_boto3_options(boto3_options)
+
+    assert options == boto3_options