Skip to content

Commit 064afaf

Browse files
committed
Enable DLQ
1 parent 3a86e36 commit 064afaf

6 files changed

Lines changed: 89 additions & 5 deletions

File tree

.devcontainer/.dev_config.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ notification_event_topic: notifications
44
notification_event_type: notification
55
service_instance_id: "001"
66
kafka_servers: ["kafka:9092"]
7+
kafka_enable_dlq: True
78
plaintext_email_template: "Dear $recipient_name,\n\n$plaintext_body\n\nWarm regards,\n\nThe GHGA Team"
89
html_email_template: '<!DOCTYPE html><html><head></head><body style="color: #00393f;padding: 12px;"><h2>Dear $recipient_name,</h2><p>$plaintext_body</p><p>Warm regards,</p><h3>The GHGA Team</h3></body></html>'
910
smtp_host: 127.0.0.1

example_config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ html_email_template: '<!DOCTYPE html><html><head></head><body style="color: #003
55
12px;"><h2>Dear $recipient_name,</h2><p>$plaintext_body</p><p>Warm regards,</p><h3>The
66
GHGA Team</h3></body></html>'
77
kafka_dlq_topic: dlq
8-
kafka_enable_dlq: false
8+
kafka_enable_dlq: true
99
kafka_max_message_size: 1048576
1010
kafka_max_retries: 0
1111
kafka_retry_backoff: 0

src/ns/inject.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
from contextlib import asynccontextmanager
2121

2222
from ghga_service_commons.utils.context import asyncnullcontext
23-
from hexkit.providers.akafka.provider import KafkaEventSubscriber
23+
from hexkit.providers.akafka.provider import KafkaEventPublisher, KafkaEventSubscriber
2424
from hexkit.providers.mongodb.provider import MongoDbDaoFactory
2525

2626
from ns.adapters.inbound.akafka import EventSubTranslator
@@ -76,7 +76,12 @@ async def prepare_event_subscriber(
7676
config=config,
7777
)
7878

79-
async with KafkaEventSubscriber.construct(
80-
config=config, translator=event_sub_translator
81-
) as event_subscriber:
79+
async with (
80+
KafkaEventPublisher.construct(config=config) as dlq_publisher,
81+
KafkaEventSubscriber.construct(
82+
config=config,
83+
translator=event_sub_translator,
84+
dlq_publisher=dlq_publisher,
85+
) as event_subscriber,
86+
):
8287
yield event_subscriber

tests/fixtures/config.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
def get_config(
3030
sources: list[BaseSettings] | None = None,
3131
default_config_yaml: Path = TEST_CONFIG_YAML,
32+
**kwargs,
3233
) -> Config:
3334
"""Merges parameters from the default TEST_CONFIG_YAML with params inferred
3435
from testcontainers.
@@ -38,6 +39,7 @@ def get_config(
3839
if sources is not None:
3940
for source in sources:
4041
sources_dict.update(**source.model_dump())
42+
sources_dict.update(**kwargs)
4143

4244
return Config(config_yaml=default_config_yaml, **sources_dict) # type: ignore
4345

tests/fixtures/test_config.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ notification_event_topic: notifications
22
notification_event_type: notification
33
service_instance_id: "001"
44
kafka_servers: ["kafka:9092"]
5+
kafka_enable_dlq: True
56
plaintext_email_template: "Dear $recipient_name,\n\n$plaintext_body\n\nWarm regards,\n\nThe GHGA Team"
67
html_email_template: '<!DOCTYPE html><html><head></head><body style="color: #00393f;padding: 12px;"><h2>Dear $recipient_name,</h2><p>$plaintext_body</p><p>Warm regards,</p><h3>The GHGA Team</h3></body></html>'
78
smtp_host: 127.0.0.1

tests/test_dlq.py

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
2+
# for the German Human Genome-Phenome Archive (GHGA)
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
"""Test to make sure that the DLQ is correctly set up for this service."""
17+
18+
import pytest
19+
from hexkit.providers.akafka.testutils import KafkaFixture
20+
21+
from ns.inject import prepare_event_subscriber
22+
from tests.fixtures.config import get_config
23+
24+
pytestmark = pytest.mark.asyncio()
25+
26+
27+
async def test_event_subscriber_dlq(kafka: KafkaFixture):
28+
"""Verify that if we get an error when consuming an event, it gets published to the DLQ."""
29+
config = get_config(sources=[kafka.config], kafka_enable_dlq=True)
30+
assert config.kafka_enable_dlq
31+
32+
# Publish an event with a bogus payload to a topic/type this service expects
33+
await kafka.publish_event(
34+
payload={"some_key": "some_value"},
35+
type_=config.notification_event_type,
36+
topic=config.notification_event_topic,
37+
key="test",
38+
)
39+
async with kafka.record_events(in_topic=config.kafka_dlq_topic) as recorder:
40+
# Consume the event, which should error and get sent to the DLQ
41+
async with prepare_event_subscriber(config=config) as event_subscriber:
42+
await event_subscriber.run(forever=False)
43+
assert recorder.recorded_events
44+
assert len(recorder.recorded_events) == 1
45+
event = recorder.recorded_events[0]
46+
assert event.key == "test"
47+
assert event.payload == {"some_key": "some_value"}
48+
49+
50+
async def test_consume_from_retry(kafka: KafkaFixture):
51+
"""Verify that this service will correctly get events from the retry topic"""
52+
config = get_config(sources=[kafka.config], kafka_enable_dlq=True)
53+
assert config.kafka_enable_dlq
54+
55+
sample_notification = {
56+
"recipient_email": "test@example.com",
57+
"email_cc": ["test2@test.com", "test3@test.com"],
58+
"email_bcc": ["test4@test.com", "test5@test.com"],
59+
"subject": "Test123",
60+
"recipient_name": "Yolanda Martinez",
61+
"plaintext_body": "Where are you, where are you, Yolanda?",
62+
}
63+
64+
# Publish an event with a proper payload to a topic/type this service expects
65+
await kafka.publish_event(
66+
payload=sample_notification,
67+
type_=config.notification_event_type,
68+
topic=config.service_name + "-retry",
69+
key="test",
70+
headers={"original_topic": config.notification_event_topic},
71+
)
72+
73+
# Consume the event
74+
async with prepare_event_subscriber(config=config) as event_subscriber:
75+
await event_subscriber.run(forever=False)

0 commit comments

Comments
 (0)