From 323df6a2046df3274eb598e801101afd7c2debfd Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Tue, 19 Apr 2022 15:51:53 +1000 Subject: [PATCH 01/12] Add support for new configs including advertised_port and use_secured_proxy. --- karapace/config.py | 2 ++ karapace/kafka_rest_apis/consumer_manager.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/karapace/config.py b/karapace/config.py index 4cc090771..01665c829 100644 --- a/karapace/config.py +++ b/karapace/config.py @@ -30,6 +30,8 @@ "group_id": "schema-registry", "host": "127.0.0.1", "port": 8081, + "advertised_port": None, + "use_secured_proxy": False, "server_tls_certfile": None, "server_tls_keyfile": None, "registry_host": "127.0.0.1", diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index 059a62e9b..69ff3f951 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -29,7 +29,11 @@ class ConsumerManager: def __init__(self, config: dict) -> None: self.config = config - self.hostname = f"http://{self.config['advertised_hostname']}:{self.config['port']}" + self.protocol = 'https' if self.config['use_secured_proxy'] else 'http' + if self.config['advertised_port'] is None: + self.hostname = f"{self.protocol}://{self.config['advertised_hostname']}:{self.config['port']}" + else: + self.hostname = f"{self.protocol}://{self.config['advertised_hostname']}:{self.config['advertised_port']}" self.log = logging.getLogger("RestConsumerManager") self.deserializer = SchemaRegistryDeserializer(config=config) self.consumers = {} From 38bd98a43163f473c8507b104d97b6d7204287a2 Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Tue, 19 Apr 2022 17:27:36 +1000 Subject: [PATCH 02/12] Updated format. --- karapace/kafka_rest_apis/consumer_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index 69ff3f951..4ade0be26 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -29,8 +29,8 @@ class ConsumerManager: def __init__(self, config: dict) -> None: self.config = config - self.protocol = 'https' if self.config['use_secured_proxy'] else 'http' - if self.config['advertised_port'] is None: + self.protocol = "https" if self.config["use_secured_proxy"] else "http" + if self.config["advertised_port"] is None: self.hostname = f"{self.protocol}://{self.config['advertised_hostname']}:{self.config['port']}" else: self.hostname = f"{self.protocol}://{self.config['advertised_hostname']}:{self.config['advertised_port']}" From ac063371feb9097f55046055066f82a4f36a4362 Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Thu, 28 Apr 2022 10:00:16 +1000 Subject: [PATCH 03/12] Change name from use_secured_proxy to advertised_protocol. --- karapace/config.py | 4 ++-- karapace/kafka_rest_apis/consumer_manager.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/karapace/config.py b/karapace/config.py index 36046baca..0b5360a1f 100644 --- a/karapace/config.py +++ b/karapace/config.py @@ -24,6 +24,8 @@ "access_logs_debug": False, "access_log_class": None, "advertised_hostname": HOSTNAME, + "advertised_port": None, + "advertised_protocol": False, "bootstrap_uri": "127.0.0.1:9092", "client_id": "sr-1", "compatibility": "BACKWARD", @@ -35,8 +37,6 @@ "group_id": "schema-registry", "host": "127.0.0.1", "port": 8081, - "advertised_port": None, - "use_secured_proxy": False, "server_tls_certfile": None, "server_tls_keyfile": None, "registry_host": "127.0.0.1", diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index 915ee3fe7..a3b0bc11d 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -34,7 +34,7 @@ def new_name() -> str: class ConsumerManager: def __init__(self, config: dict) -> None: self.config = config - self.protocol = "https" if self.config["use_secured_proxy"] else "http" + self.protocol = "https" if self.config["advertised_protocol"] else "http" if self.config["advertised_port"] is None: self.hostname = f"{self.protocol}://{self.config['advertised_hostname']}:{self.config['port']}" else: From b01eebb66f985c1ca398e5e71107b1b6378f83c0 Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Fri, 29 Apr 2022 10:12:45 +1000 Subject: [PATCH 04/12] Change the behaviour of advertised_protocol. --- karapace/config.py | 2 +- karapace/kafka_rest_apis/consumer_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/karapace/config.py b/karapace/config.py index 0b5360a1f..2ddcbe604 100644 --- a/karapace/config.py +++ b/karapace/config.py @@ -25,7 +25,7 @@ "access_log_class": None, "advertised_hostname": HOSTNAME, "advertised_port": None, - "advertised_protocol": False, + "advertised_protocol": "http", "bootstrap_uri": "127.0.0.1:9092", "client_id": "sr-1", "compatibility": "BACKWARD", diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index a3b0bc11d..aada1fef8 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -34,7 +34,7 @@ def new_name() -> str: class ConsumerManager: def __init__(self, config: dict) -> None: self.config = config - self.protocol = "https" if self.config["advertised_protocol"] else "http" + self.protocol = self.config["advertised_protocol"] if self.config["advertised_port"] is None: self.hostname = f"{self.protocol}://{self.config['advertised_hostname']}:{self.config['port']}" else: From 95da78e0f1312bd6f063348971dad0382a41cae6 Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Tue, 3 May 2022 09:51:24 +1000 Subject: [PATCH 05/12] Removing attribute self.protocol. --- karapace/kafka_rest_apis/consumer_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index aada1fef8..29362c10d 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -34,11 +34,10 @@ def new_name() -> str: class ConsumerManager: def __init__(self, config: dict) -> None: self.config = config - self.protocol = self.config["advertised_protocol"] if self.config["advertised_port"] is None: - self.hostname = f"{self.protocol}://{self.config['advertised_hostname']}:{self.config['port']}" + self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:{self.config['port']}" else: - self.hostname = f"{self.protocol}://{self.config['advertised_hostname']}:{self.config['advertised_port']}" + self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:{self.config['advertised_port']}" self.deserializer = SchemaRegistryDeserializer(config=config) self.consumers = {} self.consumer_locks = defaultdict(Lock) From 901c0e3bc815c7e7de2109b39600592cc3a95e3f Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Tue, 3 May 2022 16:55:25 +1000 Subject: [PATCH 06/12] Solving line too long error failed in CI pipeline. --- karapace/kafka_rest_apis/consumer_manager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index 29362c10d..0c6c48d8f 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -35,9 +35,11 @@ class ConsumerManager: def __init__(self, config: dict) -> None: self.config = config if self.config["advertised_port"] is None: - self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:{self.config['port']}" + self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" \ + f"{self.config['port']}" else: - self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:{self.config['advertised_port']}" + self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" \ + f"{self.config['advertised_port']}" self.deserializer = SchemaRegistryDeserializer(config=config) self.consumers = {} self.consumer_locks = defaultdict(Lock) From 7c811bb1888fc72b6312ea0e029321b86db3ea2a Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Tue, 3 May 2022 17:17:07 +1000 Subject: [PATCH 07/12] Solving line too long error failed in CI pipeline. --- karapace/kafka_rest_apis/consumer_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index 0c6c48d8f..48d62e52f 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -35,10 +35,10 @@ class ConsumerManager: def __init__(self, config: dict) -> None: self.config = config if self.config["advertised_port"] is None: - self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" \ + self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" + \ f"{self.config['port']}" else: - self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" \ + self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" + \ f"{self.config['advertised_port']}" self.deserializer = SchemaRegistryDeserializer(config=config) self.consumers = {} From c8b06cdee4161be05b4956ce2291d1ec1c90d3a2 Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Tue, 3 May 2022 17:28:29 +1000 Subject: [PATCH 08/12] Solving formatting issue using black. --- karapace/kafka_rest_apis/consumer_manager.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index 48d62e52f..766279ca4 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -35,11 +35,15 @@ class ConsumerManager: def __init__(self, config: dict) -> None: self.config = config if self.config["advertised_port"] is None: - self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" + \ - f"{self.config['port']}" + self.hostname = ( + f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" + + f"{self.config['port']}" + ) else: - self.hostname = f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" + \ - f"{self.config['advertised_port']}" + self.hostname = ( + f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" + + f"{self.config['advertised_port']}" + ) self.deserializer = SchemaRegistryDeserializer(config=config) self.consumers = {} self.consumer_locks = defaultdict(Lock) From 87e5060587a52c7deb1647c707b2f28660ebcaf5 Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Tue, 3 May 2022 17:41:52 +1000 Subject: [PATCH 09/12] Solving formatting issue using black. --- karapace/kafka_rest_apis/consumer_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index 766279ca4..d6309ce06 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -37,12 +37,12 @@ def __init__(self, config: dict) -> None: if self.config["advertised_port"] is None: self.hostname = ( f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" - + f"{self.config['port']}" + f"{self.config['port']}" ) else: self.hostname = ( f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" - + f"{self.config['advertised_port']}" + f"{self.config['advertised_port']}" ) self.deserializer = SchemaRegistryDeserializer(config=config) self.consumers = {} From fe5df0ad5c45bf49ab94adfdbf0979f5d59d9844 Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Tue, 3 May 2022 18:18:43 +1000 Subject: [PATCH 10/12] Get rid of multilines. --- karapace/kafka_rest_apis/consumer_manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index d6309ce06..615a3fb97 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -36,13 +36,11 @@ def __init__(self, config: dict) -> None: self.config = config if self.config["advertised_port"] is None: self.hostname = ( - f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" - f"{self.config['port']}" + f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:{self.config['port']}" ) else: self.hostname = ( - f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:" - f"{self.config['advertised_port']}" + f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:{self.config['advertised_port']}" ) self.deserializer = SchemaRegistryDeserializer(config=config) self.consumers = {} From 7e2082357d6f1ff92ce86d0f92f8ae8a22ce9563 Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Tue, 3 May 2022 18:24:45 +1000 Subject: [PATCH 11/12] Adding multilines back. --- karapace/kafka_rest_apis/consumer_manager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index 615a3fb97..4b7384769 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -36,11 +36,13 @@ def __init__(self, config: dict) -> None: self.config = config if self.config["advertised_port"] is None: self.hostname = ( - f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:{self.config['port']}" + f"{self.config['advertised_protocol']}://" + f"{self.config['advertised_hostname']}:{self.config['port']}" ) else: self.hostname = ( - f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:{self.config['advertised_port']}" + f"{self.config['advertised_protocol']}://" + f"{self.config['advertised_hostname']}:{self.config['advertised_port']}" ) self.deserializer = SchemaRegistryDeserializer(config=config) self.consumers = {} From 2df2cd1c4c0c737566d0e49cef5088d231ea731e Mon Sep 17 00:00:00 2001 From: Yu Wang Date: Tue, 3 May 2022 19:48:06 +1000 Subject: [PATCH 12/12] Fixing the issue by using pre-commit. --- karapace/kafka_rest_apis/consumer_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/karapace/kafka_rest_apis/consumer_manager.py index 4b7384769..ac707cb88 100644 --- a/karapace/kafka_rest_apis/consumer_manager.py +++ b/karapace/kafka_rest_apis/consumer_manager.py @@ -36,8 +36,7 @@ def __init__(self, config: dict) -> None: self.config = config if self.config["advertised_port"] is None: self.hostname = ( - f"{self.config['advertised_protocol']}://" - f"{self.config['advertised_hostname']}:{self.config['port']}" + f"{self.config['advertised_protocol']}://{self.config['advertised_hostname']}:{self.config['port']}" ) else: self.hostname = (