add error handling and code organization in multi threaded code

pyQuARC/code/checker.py

@@ -1,7 +1,7 @@
 import json
 
 from xmltodict import parse
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from .custom_checker import CustomChecker
 from .schema_validator import SchemaValidator
@@ -208,8 +208,9 @@ def _run_func(self, func, check, rule_id, metadata_content, result_dict):
             self.metadata_format, {}
         )
         with ThreadPoolExecutor() as executor:
+            futures = []
             for field_dict in list_of_fields_to_apply:
-                executor.submit(
+                future = executor.submit(
                     self._process_field,
                     func,
                     check,
@@ -219,6 +220,16 @@ def _run_func(self, func, check, rule_id, metadata_content, result_dict):
                     result_dict,
                     rule_mapping,
                 )
+                futures.append(future)
+
+            # Wait for all futures to complete
+            for future in as_completed(futures):
+                # Retrieve the result or raise an exception if an error occurred
+                try:
+                    future.result()
+                except Exception as e:
+                    # Handle the exception from the thread
+                    raise e
 
     def perform_custom_checks(self, metadata_content):
         """

pyQuARC/code/custom_checker.py

@@ -1,5 +1,5 @@
 from urllib.parse import urlparse
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 
 class CustomChecker:
@@ -181,16 +181,19 @@ def run(
                 future_results.append(future)
 
             # Retrieve results from futures
-            for future in future_results:
-                func_return = future.result()
-                valid = func_return["valid"]  # can be True, False or None
-                if valid is not None:
-                    if valid:
-                        validity = validity or (validity is None)
-                    else:
-                        if "value" in func_return:
-                            invalid_values.append(func_return["value"])
-                        validity = False
+            for future in as_completed(future_results):
+                try:
+                    func_return = future.result()
+                    valid = func_return["valid"]  # can be True, False or None
+                    if valid is not None:
+                        if valid:
+                            validity = validity or (validity is None)
+                        else:
+                            if "value" in func_return:
+                                invalid_values.append(func_return["value"])
+                            validity = False
+                except Exception as e:
+                    raise e
         result["valid"] = validity
         result["value"] = invalid_values
         return result