diff --git a/datadog_sync/model/integrations.py b/datadog_sync/model/integrations.py new file mode 100644 index 00000000..88fab7f3 --- /dev/null +++ b/datadog_sync/model/integrations.py @@ -0,0 +1,132 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +from __future__ import annotations +from typing import TYPE_CHECKING, Optional, List, Dict, Tuple, cast + +from datadog_sync.utils.base_resource import BaseResource, ResourceConfig +from datadog_sync.utils.resource_utils import CustomClientHTTPError, SkipResource + +if TYPE_CHECKING: + from datadog_sync.utils.custom_client import CustomClient + + +class Integrations(BaseResource): + resource_type = "integrations" + resource_config = ResourceConfig( + base_path="/api/v1/integration/aws", + excluded_attributes=[ + "id", + "created_at", + "modified_at", + "secret_access_key", # Security: exclude sensitive credential + ], + concurrent=True, + ) + + async def get_resources(self, client: CustomClient) -> List[Dict]: + """Get all AWS integrations from the API""" + resp = await client.get(self.resource_config.base_path) + + # The AWS integration API returns the integrations directly in the response + # We need to filter for AWS integrations and add a 'name' field for subtype identification + integrations = [] + if isinstance(resp, dict) and "accounts" in resp: + # Handle the case where AWS integrations are returned in an 'accounts' field + for account in resp.get("accounts", []): + account["name"] = "aws" # Add subtype identifier + integrations.append(account) + elif isinstance(resp, list): + # Handle the case where integrations are returned as a list + for integration in resp: + integration["name"] = "aws" # Add subtype identifier + integrations.append(integration) + elif isinstance(resp, dict): + # Handle single integration response + resp["name"] = "aws" # Add subtype identifier + integrations.append(resp) + + return integrations + + async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]: + """Import a single AWS integration by account ID""" + source_client = self.config.source_client + import_id = _id or resource.get("account_id") + + if not import_id: + raise SkipResource( + _id or "unknown", + self.resource_type, + "No account_id provided for AWS integration import" + ) + + try: + # Get the specific AWS integration configuration by account ID + resource = await source_client.get(f"{self.resource_config.base_path}/{import_id}") + + # Add subtype identifier + resource["name"] = "aws" + + except CustomClientHTTPError as err: + if err.status_code == 404: + raise SkipResource( + import_id, + self.resource_type, + f"AWS integration with account ID {import_id} not found" + ) + elif err.status_code == 403: + raise SkipResource( + import_id, + self.resource_type, + f"No access to AWS integration with account ID {import_id}" + ) + raise err + + resource = cast(dict, resource) + return import_id, resource + + async def pre_resource_action_hook(self, _id, resource: Dict) -> None: + """Hook called before any resource action""" + pass + + async def pre_apply_hook(self) -> None: + """Hook called before applying changes""" + pass + + async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: + """Create a new AWS integration in the destination organization""" + destination_client = self.config.destination_client + + # Remove the subtype identifier before sending to API + resource_copy = resource.copy() + resource_copy.pop("name", None) + + resp = await destination_client.post(self.resource_config.base_path, resource_copy) + return _id, resp + + async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]: + """Update an existing AWS integration in the destination organization""" + destination_client = self.config.destination_client + + # Remove the subtype identifier before sending to API + resource_copy = resource.copy() + resource_copy.pop("name", None) + + # Use the account_id from the stored destination resource for the PUT request + account_id = self.config.state.destination[self.resource_type][_id].get("account_id", _id) + resp = await destination_client.put( + f"{self.resource_config.base_path}/{account_id}", + resource_copy + ) + + return _id, resp + + async def delete_resource(self, _id: str) -> None: + """Delete an AWS integration from the destination organization""" + destination_client = self.config.destination_client + + # Use the account_id from the stored destination resource for the DELETE request + account_id = self.config.state.destination[self.resource_type][_id].get("account_id", _id) + await destination_client.delete(f"{self.resource_config.base_path}/{account_id}") \ No newline at end of file diff --git a/datadog_sync/models/__init__.py b/datadog_sync/models/__init__.py index 78ac3c67..bd80103b 100644 --- a/datadog_sync/models/__init__.py +++ b/datadog_sync/models/__init__.py @@ -10,6 +10,7 @@ from datadog_sync.model.downtime_schedules import DowntimeSchedules from datadog_sync.model.downtimes import Downtimes from datadog_sync.model.host_tags import HostTags +from datadog_sync.model.integrations import Integrations from datadog_sync.model.logs_archives_order import LogsArchivesOrder from datadog_sync.model.logs_archives import LogsArchives from datadog_sync.model.logs_custom_pipelines import LogsCustomPipelines diff --git a/specs/AWS_INTEGRATIONS_SPEC.md b/specs/AWS_INTEGRATIONS_SPEC.md new file mode 100644 index 00000000..1de4cecb --- /dev/null +++ b/specs/AWS_INTEGRATIONS_SPEC.md @@ -0,0 +1,256 @@ +# AWS Integration Sync Capability - Technical Specification + +## 1. Overview + +This specification defines the requirements and implementation plan for adding AWS Integration configuration synchronization capabilities to the datadog-sync-cli tool. This will allow users to sync AWS Integration configurations between Datadog organizations using the existing `import`, `sync`, `migrate`, and `reset` commands. + +## 2. Functional Requirements + +### 2.1 Core Resource Support +- **FR-1**: Add a new resource type named `integrations` to the datadog-sync-cli +- **FR-2**: Support a new subtype (name) called `aws` for the `integrations` resource +- **FR-3**: Enable full CRUD operations for AWS Integration configurations through all existing commands + +### 2.2 Command Support +- **FR-4**: Support `import` command to retrieve AWS Integration configurations from source organization +- **FR-5**: Support `sync` command to create/update AWS Integration configurations in destination organization +- **FR-6**: Support `migrate` command to perform import followed by sync in one operation +- **FR-7**: Support `reset` command to delete AWS Integration configurations from destination organization +- **FR-8**: Support `diffs` command to show differences between source and destination AWS Integration configurations + +### 2.3 Resource Management +- **FR-9**: Handle AWS Account ID as the primary identifier for AWS Integration resources +- **FR-10**: Support filtering AWS Integration resources by Account ID, region, or other attributes +- **FR-11**: Maintain resource relationships and dependencies with existing resources (if any) + +### 2.4 Configuration Synchronization +- **FR-12**: Sync all AWS Integration configuration parameters including: + - Account ID and role name + - Access credentials (for GovCloud/China accounts) + - Metrics collection settings + - Resource collection settings + - Tag filters and host tags + - Excluded regions + - Namespace-specific rules + - CSPM resource collection settings + +## 3. Non-Functional Requirements + +### 3.1 Performance +- **NFR-1**: AWS Integration operations should complete within reasonable time limits (< 30 seconds for typical configurations) +- **NFR-2**: Support concurrent processing when multiple AWS accounts are configured +- **NFR-3**: Implement appropriate rate limiting to respect Datadog API limits + +### 3.2 Reliability +- **NFR-4**: Handle API errors gracefully with appropriate retry logic +- **NFR-5**: Provide clear error messages for common failure scenarios (authentication, permissions, etc.) +- **NFR-6**: Ensure idempotent operations that can be safely retried + +### 3.3 Security +- **NFR-7**: Handle AWS credentials securely, never logging or exposing sensitive values +- **NFR-8**: Validate required permissions before attempting operations +- **NFR-9**: Follow security best practices for API communication + +### 3.4 Compatibility +- **NFR-10**: Maintain backward compatibility with existing datadog-sync-cli functionality +- **NFR-11**: Support all Datadog regions and API endpoints +- **NFR-12**: Follow existing code patterns and conventions used in the codebase + +## 4. API Integration Details + +### 4.1 Datadog API Endpoints +The implementation will use the following Datadog API endpoints: + +- **GET/POST/PUT/DELETE** `/api/v1/integration/aws` - Main AWS Integration configuration +- **GET/POST** `/api/v1/integration/aws/filtering` - AWS tag filtering rules +- **GET** `/api/v2/integration/aws/available_namespaces` - Available CloudWatch namespaces +- **GET/POST/DELETE** `/api/v1/integration/aws/logs` - AWS Logs integration configuration + +### 4.2 Resource Schema +Based on the API documentation, the AWS Integration resource schema includes: + +```json +{ + "account_id": "string (required)", + "role_name": "string (required)", + "external_id": "string (optional)", + "filter_tags": ["string (optional)"], + "host_tags": ["string (optional)"], + "account_specific_namespace_rules": { + "namespace": "boolean" + }, + "excluded_regions": ["string (optional)"], + "metrics_collection_enabled": "boolean (optional)", + "cspm_resource_collection_enabled": "boolean (optional)", + "extended_resource_collection_enabled": "boolean (optional)", + "access_key_id": "string (optional, GovCloud/China only)", + "secret_access_key": "string (optional, GovCloud/China only)" +} +``` + +## 5. Implementation Plan + +### 5.1 File Structure +Following the existing pattern, create these files: + +``` +datadog_sync/model/integrations.py # Main integrations resource class +tests/integration/resources/test_integrations.py # Integration tests +tests/unit/model/test_integrations.py # Unit tests +``` + +### 5.2 Resource Implementation Details + +#### 5.2.1 Base Resource Class +Create `Integrations` class extending `BaseResource`: + +```python +class Integrations(BaseResource): + resource_type = "integrations" + resource_config = ResourceConfig( + base_path="/api/v1/integration/aws", + excluded_attributes=[ + "id", + "created_at", + "modified_at", + "secret_access_key" # Security: exclude sensitive fields + ], + concurrent=True + ) +``` + +#### 5.2.2 Subtype Handling +Implement subtype filtering to support the `aws` subtype: +- Filter resources by integration type during import +- Handle AWS-specific API endpoints and schemas +- Support future extensibility for other integration types (azure, gcp, etc.) + +#### 5.2.3 Key Methods Implementation + +**get_resources()**: Retrieve all AWS integrations from the API +**import_resource()**: Fetch individual AWS integration configuration +**create_resource()**: Create new AWS integration in destination org +**update_resource()**: Update existing AWS integration configuration +**delete_resource()**: Remove AWS integration from destination org + +### 5.3 Testing Strategy + +#### 5.3.1 Unit Tests +- Test resource CRUD operations +- Test filtering and subtype handling +- Test error handling and edge cases +- Mock API responses for isolated testing + +#### 5.3.2 Integration Tests +- End-to-end testing with real API calls (using VCR cassettes) +- Test all commands (import, sync, migrate, reset, diffs) +- Test filtering and resource management +- Test error scenarios and recovery + +#### 5.3.3 Test Cases +- Import AWS integrations from source organization +- Sync AWS integrations to destination organization +- Update existing AWS integration configurations +- Delete AWS integrations with reset command +- Filter AWS integrations by account ID or region +- Handle API errors and retry scenarios +- Validate credential security and handling + +## 6. Configuration and Usage + +### 6.1 Command Examples + +```bash +# Import all integrations (including AWS) +datadog-sync import --resources="integrations" \ + --source-api-key="..." --source-app-key="..." + +# Import only AWS integrations +datadog-sync import --resources="integrations" \ + --filter='Type=integrations;Name=name;Value=aws' \ + --source-api-key="..." --source-app-key="..." + +# Sync AWS integrations to destination +datadog-sync sync --resources="integrations" \ + --destination-api-key="..." --destination-app-key="..." + +# Show differences in AWS integrations +datadog-sync diffs --resources="integrations" \ + --destination-api-key="..." --destination-app-key="..." +``` + +### 6.2 Filtering Support +- Filter by integration name (subtype): `--filter='Type=integrations;Name=name;Value=aws'` +- Filter by AWS Account ID: `--filter='Type=integrations;Name=account_id;Value=123456789012'` +- Filter by region: `--filter='Type=integrations;Name=excluded_regions;Value=us-west-2'` + +## 7. Dependencies and Prerequisites + +### 7.1 Required Permissions +The following Datadog permissions are required: +- `aws_configuration_read` - For import and diffs operations +- `aws_configuration_edit` - For sync and reset operations + +### 7.2 External Dependencies +- No new external Python dependencies required +- Leverage existing datadog-api-client functionality +- Use existing HTTP client and retry mechanisms + +## 8. Risk Assessment and Mitigation + +### 8.1 Technical Risks +- **Risk**: API rate limiting during bulk operations + - **Mitigation**: Implement backoff and retry logic, support concurrent limits +- **Risk**: Credential handling and security + - **Mitigation**: Follow existing patterns, exclude sensitive fields from logs/state +- **Risk**: Complex AWS account dependencies + - **Mitigation**: Clear error messages, validation of prerequisites + +### 8.2 Operational Risks +- **Risk**: Breaking existing functionality + - **Mitigation**: Comprehensive testing, follow existing patterns exactly +- **Risk**: Data loss during sync operations + - **Mitigation**: Implement backup functionality similar to reset command + +## 9. Success Criteria + +### 9.1 Functional Success Criteria +- [ ] All four commands (import, sync, migrate, reset) work correctly for AWS integrations +- [ ] Filtering works properly for integrations resource and aws subtype +- [ ] AWS Integration configurations sync accurately between organizations +- [ ] Error handling provides clear, actionable messages + +### 9.2 Quality Success Criteria +- [ ] Unit test coverage >= 95% for new code +- [ ] Integration tests pass for all major scenarios +- [ ] Code review approval from maintainers +- [ ] Documentation updated to include integrations resource + +### 9.3 Performance Success Criteria +- [ ] Import/sync operations complete within 30 seconds for typical configurations +- [ ] No performance regression in existing functionality +- [ ] Memory usage remains within acceptable limits + +## 10. Future Enhancements + +### 10.1 Additional Integration Types +The `integrations` resource is designed to be extensible for future integration types: +- Azure Integration (`name=azure`) +- Google Cloud Integration (`name=gcp`) +- Other cloud or service integrations + +### 10.2 Enhanced Filtering +- Support for more granular filtering options +- Regex-based filtering for complex scenarios +- Multi-attribute filtering combinations + +### 10.3 Advanced Features +- Validation of AWS IAM roles and permissions +- Automatic dependency resolution for related resources +- Bulk operations optimization for large numbers of integrations + +--- + +**Document Version**: 1.0 +**Created**: 2024-11-04 +**Status**: Ready for Implementation Review \ No newline at end of file diff --git a/tests/integration/resources/test_integrations.py b/tests/integration/resources/test_integrations.py new file mode 100644 index 00000000..409beb8d --- /dev/null +++ b/tests/integration/resources/test_integrations.py @@ -0,0 +1,24 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +from datadog_sync.models import Integrations +from tests.integration.helpers import BaseResourcesTestClass + + +class TestIntegrationsResources(BaseResourcesTestClass): + resource_type = Integrations.resource_type + dependencies = [] # AWS integrations typically have no dependencies on other resources + field_to_update = "role_name" # Field that can be safely updated for testing + force_missing_deps = False # No dependencies to force + + @staticmethod + def compute_cleanup_changes(resource_count, num_of_skips): + """AWS integrations cleanup logic""" + return resource_count + + @staticmethod + def compute_import_changes(resource_count, num_of_skips): + """AWS integrations import logic""" + return resource_count \ No newline at end of file diff --git a/tests/unit/model/test_integrations.py b/tests/unit/model/test_integrations.py new file mode 100644 index 00000000..76842021 --- /dev/null +++ b/tests/unit/model/test_integrations.py @@ -0,0 +1,219 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the 3-clause BSD style license (see LICENSE). +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +import pytest +from unittest.mock import AsyncMock, MagicMock + +from datadog_sync.model.integrations import Integrations +from datadog_sync.utils.resource_utils import CustomClientHTTPError, SkipResource + + +@pytest.fixture +def mock_config(): + """Create a mock configuration""" + config = MagicMock() + config.source_client = AsyncMock() + config.destination_client = AsyncMock() + config.state = MagicMock() + config.state.destination = {"integrations": {}} + return config + + +@pytest.fixture +def integrations_resource(mock_config): + """Create an Integrations resource instance""" + return Integrations(mock_config) + + +class TestIntegrationsResource: + def test_resource_type(self, integrations_resource): + """Test that resource type is correct""" + assert integrations_resource.resource_type == "integrations" + + def test_resource_config(self, integrations_resource): + """Test resource configuration""" + config = integrations_resource.resource_config + assert config.base_path == "/api/v1/integration/aws" + assert "secret_access_key" in config.excluded_attributes + assert "id" in config.excluded_attributes + assert config.concurrent is True + + @pytest.mark.asyncio + async def test_get_resources_with_accounts(self, integrations_resource, mock_config): + """Test get_resources with accounts response""" + mock_response = { + "accounts": [ + {"account_id": "123456789012", "role_name": "DatadogIntegrationRole"}, + {"account_id": "987654321098", "role_name": "DatadogIntegrationRole"} + ] + } + mock_config.source_client.get.return_value = mock_response + + result = await integrations_resource.get_resources(mock_config.source_client) + + assert len(result) == 2 + assert all(resource["name"] == "aws" for resource in result) + assert result[0]["account_id"] == "123456789012" + assert result[1]["account_id"] == "987654321098" + + @pytest.mark.asyncio + async def test_get_resources_with_list_response(self, integrations_resource, mock_config): + """Test get_resources with list response""" + mock_response = [ + {"account_id": "123456789012", "role_name": "DatadogIntegrationRole"}, + {"account_id": "987654321098", "role_name": "DatadogIntegrationRole"} + ] + mock_config.source_client.get.return_value = mock_response + + result = await integrations_resource.get_resources(mock_config.source_client) + + assert len(result) == 2 + assert all(resource["name"] == "aws" for resource in result) + + @pytest.mark.asyncio + async def test_get_resources_with_single_response(self, integrations_resource, mock_config): + """Test get_resources with single dict response""" + mock_response = {"account_id": "123456789012", "role_name": "DatadogIntegrationRole"} + mock_config.source_client.get.return_value = mock_response + + result = await integrations_resource.get_resources(mock_config.source_client) + + assert len(result) == 1 + assert result[0]["name"] == "aws" + assert result[0]["account_id"] == "123456789012" + + @pytest.mark.asyncio + async def test_import_resource_success(self, integrations_resource, mock_config): + """Test successful import of AWS integration""" + account_id = "123456789012" + mock_response = {"account_id": account_id, "role_name": "DatadogIntegrationRole"} + mock_config.source_client.get.return_value = mock_response + + result_id, result_resource = await integrations_resource.import_resource(_id=account_id) + + assert result_id == account_id + assert result_resource["name"] == "aws" + assert result_resource["account_id"] == account_id + mock_config.source_client.get.assert_called_once_with("/api/v1/integration/aws/123456789012") + + @pytest.mark.asyncio + async def test_import_resource_with_resource_dict(self, integrations_resource, mock_config): + """Test import when resource dict is provided""" + resource_dict = {"account_id": "123456789012"} + mock_response = {"account_id": "123456789012", "role_name": "DatadogIntegrationRole"} + mock_config.source_client.get.return_value = mock_response + + result_id, result_resource = await integrations_resource.import_resource(resource=resource_dict) + + assert result_id == "123456789012" + assert result_resource["name"] == "aws" + + @pytest.mark.asyncio + async def test_import_resource_no_account_id(self, integrations_resource): + """Test import with no account ID provided""" + with pytest.raises(SkipResource) as exc_info: + await integrations_resource.import_resource() + + assert "No account_id provided" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_import_resource_not_found(self, integrations_resource, mock_config): + """Test import when AWS integration is not found""" + account_id = "999999999999" + mock_config.source_client.get.side_effect = CustomClientHTTPError( + status_code=404, + message="Not found" + ) + + with pytest.raises(SkipResource) as exc_info: + await integrations_resource.import_resource(_id=account_id) + + assert "not found" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_import_resource_forbidden(self, integrations_resource, mock_config): + """Test import when access is forbidden""" + account_id = "123456789012" + mock_config.source_client.get.side_effect = CustomClientHTTPError( + status_code=403, + message="Forbidden" + ) + + with pytest.raises(SkipResource) as exc_info: + await integrations_resource.import_resource(_id=account_id) + + assert "No access to" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_import_resource_other_http_error(self, integrations_resource, mock_config): + """Test import with other HTTP errors""" + account_id = "123456789012" + mock_config.source_client.get.side_effect = CustomClientHTTPError( + status_code=500, + message="Internal Server Error" + ) + + with pytest.raises(CustomClientHTTPError): + await integrations_resource.import_resource(_id=account_id) + + @pytest.mark.asyncio + async def test_create_resource(self, integrations_resource, mock_config): + """Test creating a new AWS integration""" + resource = { + "account_id": "123456789012", + "role_name": "DatadogIntegrationRole", + "name": "aws" # This should be removed before API call + } + mock_response = {"account_id": "123456789012", "role_name": "DatadogIntegrationRole"} + mock_config.destination_client.post.return_value = mock_response + + result_id, result_resource = await integrations_resource.create_resource("123456789012", resource) + + assert result_id == "123456789012" + # Verify the API call was made without the 'name' field + called_resource = mock_config.destination_client.post.call_args[0][1] + assert "name" not in called_resource + assert called_resource["account_id"] == "123456789012" + + @pytest.mark.asyncio + async def test_update_resource(self, integrations_resource, mock_config): + """Test updating an AWS integration""" + # Mock existing resource in destination state + mock_config.state.destination["integrations"]["123456789012"] = { + "account_id": "123456789012" + } + + resource = { + "account_id": "123456789012", + "role_name": "UpdatedDatadogRole", + "name": "aws" # This should be removed before API call + } + mock_response = {"account_id": "123456789012", "role_name": "UpdatedDatadogRole"} + mock_config.destination_client.put.return_value = mock_response + + result_id, result_resource = await integrations_resource.update_resource("123456789012", resource) + + assert result_id == "123456789012" + # Verify the API call was made without the 'name' field and used correct URL + mock_config.destination_client.put.assert_called_once() + called_url = mock_config.destination_client.put.call_args[0][0] + called_resource = mock_config.destination_client.put.call_args[0][1] + assert "/api/v1/integration/aws/123456789012" in called_url + assert "name" not in called_resource + + @pytest.mark.asyncio + async def test_delete_resource(self, integrations_resource, mock_config): + """Test deleting an AWS integration""" + # Mock existing resource in destination state + mock_config.state.destination["integrations"]["123456789012"] = { + "account_id": "123456789012" + } + + await integrations_resource.delete_resource("123456789012") + + # Verify the API call was made with correct URL + mock_config.destination_client.delete.assert_called_once_with( + "/api/v1/integration/aws/123456789012" + ) \ No newline at end of file