Skip to content

Commit 20f8be0

Browse files
committed
Workaround import pyspark error caused by socketserver.UnixStreamServer not present on Windows
1 parent bcbf7d6 commit 20f8be0

File tree

1 file changed

+30
-15
lines changed

1 file changed

+30
-15
lines changed

python/pyspark/accumulators.py

Lines changed: 30 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -321,21 +321,36 @@ def shutdown(self) -> None:
321321
self.server_close()
322322

323323

324-
class AccumulatorUnixServer(socketserver.UnixStreamServer):
325-
server_shutdown = False
326-
327-
def __init__(
328-
self, socket_path: str, RequestHandlerClass: Type[socketserver.BaseRequestHandler]
329-
):
330-
super().__init__(socket_path, RequestHandlerClass)
331-
self.auth_token = None
332-
333-
def shutdown(self) -> None:
334-
self.server_shutdown = True
335-
super().shutdown()
336-
self.server_close()
337-
if os.path.exists(self.server_address): # type: ignore[arg-type]
338-
os.remove(self.server_address) # type: ignore[arg-type]
324+
# socketserver.UnixStreamServer is not available on Windows yet
325+
# (https://github.com/python/cpython/issues/77589).
326+
if hasattr(socketserver, "UnixStreamServer"):
327+
328+
class AccumulatorUnixServer(socketserver.UnixStreamServer):
329+
server_shutdown = False
330+
331+
def __init__(
332+
self, socket_path: str, RequestHandlerClass: Type[socketserver.BaseRequestHandler]
333+
):
334+
super().__init__(socket_path, RequestHandlerClass)
335+
self.auth_token = None
336+
337+
def shutdown(self) -> None:
338+
self.server_shutdown = True
339+
super().shutdown()
340+
self.server_close()
341+
if os.path.exists(self.server_address): # type: ignore[arg-type]
342+
os.remove(self.server_address) # type: ignore[arg-type]
343+
344+
else:
345+
346+
class AccumulatorUnixServer(socketserver.TCPServer): # type: ignore[no-redef]
347+
def __init__(
348+
self, socket_path: str, RequestHandlerClass: Type[socketserver.BaseRequestHandler]
349+
):
350+
raise NotImplementedError(
351+
"Unix Domain Sockets are not supported on this platform. "
352+
"Please disable it by setting spark.python.unix.domain.socket.enabled to false."
353+
)
339354

340355

341356
def _start_update_server(

0 commit comments

Comments
 (0)