fix!: Ensure all open uses UTF-8 instead of default encoding.

Signed-off-by: Karthik Bekal Pattathana <[email protected]>

Author: karthikbekalp

.github/scripts/get_latest_changelog.py

@@ -32,7 +32,7 @@
 import re
 
 h2 = r"^##\s.*$"
-with open("CHANGELOG.md") as f:
+with open("CHANGELOG.md", encoding="utf-8") as f:
     contents = f.read()
 matches = re.findall(h2, contents, re.MULTILINE)
 changelog = contents[: contents.find(matches[1]) - 1] if len(matches) > 1 else contents

src/openjd/adaptor_runtime/_background/backend_runner.py

@@ -116,7 +116,9 @@ def run(self, *, on_connection_file_written: List[Callable[[], None]] | None = N
             raise
 
         try:
-            with secure_open(self._connection_file_path, open_mode="w") as conn_file:
+            with secure_open(
+                self._connection_file_path, open_mode="w", encoding="utf-8"
+            ) as conn_file:
                 json.dump(
                     ConnectionSettings(server_path),
                     conn_file,

src/openjd/adaptor_runtime/_background/frontend_runner.py

@@ -153,7 +153,7 @@ def init(
         bootstrap_output_path = os.path.join(
             bootstrap_log_dir, f"adaptor-runtime-background-bootstrap-output-{bootstrap_id}.log"
         )
-        output_log_file = open(bootstrap_output_path, mode="w+")
+        output_log_file = open(bootstrap_output_path, mode="w+", encoding="utf-8")
         try:
             process = subprocess.Popen(
                 args,
@@ -194,7 +194,7 @@ def init(
             if process.stderr:
                 process.stderr.close()
 
-            with open(bootstrap_output_path, mode="r") as f:
+            with open(bootstrap_output_path, mode="r", encoding="utf-8") as f:
                 bootstrap_output = f.readlines()
             _logger.info("========== BEGIN BOOTSTRAP OUTPUT CONTENTS ==========")
             for line in bootstrap_output:
@@ -203,7 +203,7 @@ def init(
 
             _logger.info(f"Checking for bootstrap logs at '{bootstrap_log_path}'")
             try:
-                with open(bootstrap_log_path, mode="r") as f:
+                with open(bootstrap_log_path, mode="r", encoding="utf-8") as f:
                     bootstrap_logs = f.readlines()
             except Exception as e:
                 _logger.error(f"Failed to get bootstrap logs at '{bootstrap_log_path}': {e}")
@@ -442,7 +442,8 @@ def _wait_for_connection_file(
 
     def file_is_openable() -> bool:
         try:
-            open(filepath, mode="r").close()
+            with open(filepath, mode="r", encoding="utf-8"):
+                pass
         except IOError:
             # File is not available yet
             return False

src/openjd/adaptor_runtime/_background/loaders.py

@@ -35,7 +35,7 @@ class ConnectionSettingsFileLoader(ConnectionSettingsLoader):
 
     def load(self) -> ConnectionSettings:
         try:
-            with open(self.file_path) as conn_file:
+            with open(self.file_path, encoding="utf-8") as conn_file:
                 loaded_settings = json.load(conn_file)
         except OSError as e:
             errmsg = f"Failed to open connection file '{self.file_path}': {e}"

src/openjd/adaptor_runtime/_background/log_buffers.py

@@ -132,7 +132,7 @@ def __init__(self, filepath: str, *, formatter: logging.Formatter | None = None)
     def buffer(self, record: logging.LogRecord) -> None:
         with (
             self._file_lock,
-            secure_open(self._filepath, open_mode="a") as f,
+            secure_open(self._filepath, open_mode="a", encoding="utf-8") as f,
         ):
             f.write(self._format(record))
 
@@ -142,7 +142,7 @@ def chunk(self) -> BufferedOutput:
         with (
             self._chunk_lock,
             self._file_lock,
-            open(self._filepath, mode="r") as f,
+            open(self._filepath, mode="r", encoding="utf-8") as f,
         ):
             self._chunk.id = id
             f.seek(self._chunk.start)

src/openjd/adaptor_runtime/_entrypoint.py

@@ -179,6 +179,10 @@ def _init_loggers(self, *, bootstrap_log_path: str | None = None) -> _LogConfig:
         formatter = ConditionalFormatter(
             "%(levelname)s: %(message)s", ignore_patterns=[_OPENJD_LOG_REGEX]
         )
+
+        # Ensure stdout uses UTF-8 encoding to avoid issues with
+        # encoding Non-ASCII characters.
+        sys.stdout.reconfigure(encoding="utf-8")  # type: ignore
         stream_handler = logging.StreamHandler(sys.stdout)
         stream_handler.setFormatter(formatter)
 
@@ -590,14 +594,14 @@ def _load_data(data: str) -> dict:
 
 def _load_yaml_json(data: str) -> Any:
     """
-    Loads a YAML/JSON file/string.
+    Loads a YAML/JSON file/string using the UTF-8 encoding.
 
     Note that yaml.safe_load() is capable of loading JSON documents.
     """
     loaded_yaml = None
     if data.startswith("file://"):
         filepath = data[len("file://") :]
-        with open(filepath) as yaml_file:
+        with open(filepath, encoding="utf-8") as yaml_file:
             loaded_yaml = yaml.safe_load(yaml_file)
     else:
         loaded_yaml = yaml.safe_load(data)

src/openjd/adaptor_runtime/adaptors/_validator.py

@@ -74,7 +74,7 @@ def from_schema_file(schema_path: str) -> AdaptorDataValidator:
             schema_path (str): The path to the JSON schema file to use.
         """
         try:
-            with open(schema_path) as schema_file:
+            with open(schema_path, encoding="utf-8") as schema_file:
                 schema = json.load(schema_file)
         except json.JSONDecodeError as e:
             _logger.error(f"Failed to decode JSON schema file: {e}")
@@ -148,7 +148,7 @@ def _load_yaml_json(data: str) -> Any:
     loaded_yaml = None
     if data.startswith("file://"):
         filepath = data[len("file://") :]
-        with open(filepath) as yaml_file:
+        with open(filepath, encoding="utf-8") as yaml_file:
             loaded_yaml = yaml.safe_load(yaml_file)
     else:
         loaded_yaml = yaml.safe_load(data)

src/openjd/adaptor_runtime/adaptors/configuration/_configuration.py

@@ -82,7 +82,8 @@ def from_file(
         """
 
         try:
-            config = json.load(open(config_path))
+            with open(config_path, encoding="utf-8") as config_file:
+                config = json.load(config_file)
         except OSError as e:
             _logger.error(f"Failed to open configuration at {config_path}: {e}")
             raise
@@ -102,7 +103,8 @@ def from_file(
         schema_paths = schema_path if isinstance(schema_path, list) else [schema_path]
         for path in schema_paths:
             try:
-                schema = json.load(open(path))
+                with open(path, encoding="utf-8") as schema_file:
+                    schema = json.load(schema_file)
             except OSError as e:
                 _logger.error(f"Failed to open configuration schema at {path}: {e}")
                 raise

src/openjd/adaptor_runtime/adaptors/configuration/_configuration_manager.py

@@ -95,7 +95,7 @@ def _ensure_config_file(filepath: str, *, create: bool = False) -> bool:
         _logger.info(f"Creating empty configuration at {filepath}")
         try:
             os.makedirs(os.path.dirname(filepath), mode=stat.S_IRWXU, exist_ok=True)
-            with secure_open(filepath, open_mode="w") as f:
+            with secure_open(filepath, open_mode="w", encoding="utf-8") as f:
                 json.dump({}, f)
         except OSError as e:
             _logger.warning(f"Could not write empty configuration to {filepath}: {e}")

test/openjd/adaptor_runtime/integ/_utils/test_secure_open.py

@@ -24,7 +24,7 @@ def create_file():
         f"secure_open_test_{''.join(random.choice(string.ascii_letters) for _ in range(10))}.txt"
     )
     test_file_path = os.path.join(tempfile.gettempdir(), test_file_name)
-    with secure_open(test_file_path, open_mode="w") as test_file:
+    with secure_open(test_file_path, open_mode="w", encoding="utf-8") as test_file:
         test_file.write(file_content)
     yield test_file_path, file_content
     os.remove(test_file_path)
@@ -36,7 +36,7 @@ def test_secure_open_write_and_read(self, create_file):
         Test if the file owner can write and read the file
         """
         test_file_path, file_content = create_file
-        with secure_open(test_file_path, open_mode="r") as test_file:
+        with secure_open(test_file_path, open_mode="r", encoding="utf-8") as test_file:
             result = test_file.read()
         assert result == file_content
 
@@ -61,7 +61,7 @@ def test_secure_open_file_windows_permission(self, create_file, win_test_user):
 
         try:
             with pytest.raises(PermissionError):
-                with open(test_file_path, "r") as f:
+                with open(test_file_path, "r", encoding="utf-8") as f:
                     f.read()
         finally:
             # Revert the impersonation