From 1848ea3b5d0683955354a1f6b11d1e35d729ac47 Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Thu, 27 Nov 2025 16:37:40 +0100 Subject: [PATCH 1/8] feat(ray): better handle the absence of ray.init() --- ddtrace/contrib/internal/ray/__init__.py | 13 +- ddtrace/contrib/internal/ray/patch.py | 3 +- tests/contrib/ray/jobs/actor_and_task.py | 62 --- tests/contrib/ray/jobs/actor_interactions.py | 49 -- tests/contrib/ray/jobs/actor_without_init.py | 24 + tests/contrib/ray/jobs/error_in_task.py | 21 - tests/contrib/ray/jobs/nested_tasks.py | 27 -- tests/contrib/ray/jobs/simple_actor.py | 37 -- tests/contrib/ray/jobs/simple_task.py | 20 - tests/contrib/ray/jobs/simple_wait.py | 20 - tests/contrib/ray/jobs/task_without_init.py | 25 ++ tests/contrib/ray/test_ray.py | 85 +++- ...ayWithoutInit.test_actor_without_init.json | 227 ++++++++++ ...RayWithoutInit.test_task_without_init.json | 424 ++++++++++++++++++ 14 files changed, 783 insertions(+), 254 deletions(-) delete mode 100644 tests/contrib/ray/jobs/actor_and_task.py delete mode 100644 tests/contrib/ray/jobs/actor_interactions.py create mode 100644 tests/contrib/ray/jobs/actor_without_init.py delete mode 100644 tests/contrib/ray/jobs/error_in_task.py delete mode 100644 tests/contrib/ray/jobs/nested_tasks.py delete mode 100644 tests/contrib/ray/jobs/simple_actor.py delete mode 100644 tests/contrib/ray/jobs/simple_task.py delete mode 100644 tests/contrib/ray/jobs/simple_wait.py create mode 100644 tests/contrib/ray/jobs/task_without_init.py create mode 100644 tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json create mode 100644 tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json diff --git a/ddtrace/contrib/internal/ray/__init__.py b/ddtrace/contrib/internal/ray/__init__.py index e188852cd50..b2499ded631 100644 --- a/ddtrace/contrib/internal/ray/__init__.py +++ b/ddtrace/contrib/internal/ray/__init__.py @@ -11,16 +11,19 @@ you have to follow one of the two methods below: The recommended way to instrument Ray, is to instrument the Ray cluster. -You can do it by starting the Ray head with a tracing startup hook:: - ray start --head --tracing-startup-hook=ddtrace.contrib.ray:setup_tracing +You can do it by starting the Ray head using ddtrace-run:: -Otherwise, you can specify the tracing hook in `ray.init()` using:: + DD_PATCH_MODULES="ray:true, aiohttp:false, grpc:false, requests:false" ddtrace-run ray start --head - ray.init(_tracing_startup_hook="ddtrace.contrib.ray:setup_tracing") +DD_PATCH_MODULES will allow to reduce noise. -Note that this method does not provide full tracing capabilities. +You can do it by starting the Ray head with a tracing startup hook:: + + ray start --head --tracing-startup-hook=ddtrace.contrib.ray:setup_tracing +Note that this method does not provide full tracing capabilities if ``ray.init()`` is not called at the top +of your job scripts. Configuration ~~~~~~~~~~~~~ diff --git a/ddtrace/contrib/internal/ray/patch.py b/ddtrace/contrib/internal/ray/patch.py index 6e607286d16..9957944f390 100644 --- a/ddtrace/contrib/internal/ray/patch.py +++ b/ddtrace/contrib/internal/ray/patch.py @@ -153,11 +153,10 @@ def traced_submit_task(wrapped, instance, args, kwargs): # This is done under a lock as multiple task could be submit at the same time # and thus try to modify the signature as the same time with instance._inject_lock: - if not getattr(instance._function, "_dd_trace_wrapped", False): + if instance._function_signature is None: instance._function = _wrap_remote_function_execution(instance._function) instance._function.__signature__ = _inject_dd_trace_ctx_kwarg(instance._function) instance._function_signature = extract_signature(instance._function) - instance._function._dd_trace_wrapped = True with tracer.trace( "task.submit", diff --git a/tests/contrib/ray/jobs/actor_and_task.py b/tests/contrib/ray/jobs/actor_and_task.py deleted file mode 100644 index 00758bbbfc7..00000000000 --- a/tests/contrib/ray/jobs/actor_and_task.py +++ /dev/null @@ -1,62 +0,0 @@ -import ray - - -ray.init(logging_level=0) - - -@ray.remote -def compute_value(x): - return x + 1 - - -@ray.remote -def batch_compute(values): - futures = [compute_value.remote(val) for val in values] - return ray.get(futures) - - -@ray.remote -class ComputationManager: - def __init__(self): - self.computation_count = 0 - self.results = [] - - def increment_count(self): - self.computation_count += 1 - return self.computation_count - - def get_count(self): - return self.computation_count - - def add_result(self, result): - self.results.append(result) - return len(self.results) - - def get_results(self): - return self.results - - def compute_and_store(self, values): - self.increment_count() - - future = batch_compute.remote(values) - results = ray.get(future) - - for result in results: - self.add_result(result) - - return {"computation_count": self.get_count(), "results": set(results), "total_stored": len(self.get_results())} - - -def main(): - manager = ComputationManager.remote() - result = ray.get(manager.compute_and_store.remote([2, 3, 4])) - assert result == { - "computation_count": 1, - "results": {3, 4, 5}, - "total_stored": 3, - }, f"Unexpected results: {result['results']}" - - -if __name__ == "__main__": - main() - ray.shutdown() diff --git a/tests/contrib/ray/jobs/actor_interactions.py b/tests/contrib/ray/jobs/actor_interactions.py deleted file mode 100644 index ed40e8bfb1f..00000000000 --- a/tests/contrib/ray/jobs/actor_interactions.py +++ /dev/null @@ -1,49 +0,0 @@ -import ray - - -ray.init() - - -@ray.remote -class Sender: - def __init__(self): - self.sent_count = 0 - - def send_message(self, receiver, message): - self.sent_count += 1 - future = receiver.receive_message.remote(message) - return ray.get(future) - - def get_sent_count(self): - return self.sent_count - - -@ray.remote -class Receiver: - def __init__(self): - self.received_messages = [] - - def receive_message(self, message): - self.received_messages.append(message) - return f"received: {message}" - - def get_messages(self): - return self.received_messages - - -def main(): - sender = Sender.remote() - receiver = Receiver.remote() - - result = ray.get(sender.send_message.remote(receiver, "hello")) - sent_count = ray.get(sender.get_sent_count.remote()) - messages = ray.get(receiver.get_messages.remote()) - - assert result == "received: hello" - assert sent_count == 1 - assert messages == ["hello"] - - -if __name__ == "__main__": - main() - ray.shutdown() diff --git a/tests/contrib/ray/jobs/actor_without_init.py b/tests/contrib/ray/jobs/actor_without_init.py new file mode 100644 index 00000000000..b107df30084 --- /dev/null +++ b/tests/contrib/ray/jobs/actor_without_init.py @@ -0,0 +1,24 @@ +import ray + + +@ray.remote +class Counter: + def __init__(self): + self.value = 0 + + def increment(self): + self.value += 1 + return self.value + + def get_value(self): + return self.value + + +def main(): + counter = Counter.remote() + result = ray.get(counter.increment.remote()) + assert result == 1, f"Expected 1, got {result}" + + +if __name__ == "__main__": + main() diff --git a/tests/contrib/ray/jobs/error_in_task.py b/tests/contrib/ray/jobs/error_in_task.py deleted file mode 100644 index 1cef49bdb5e..00000000000 --- a/tests/contrib/ray/jobs/error_in_task.py +++ /dev/null @@ -1,21 +0,0 @@ -import ray - - -ray.init() - - -@ray.remote -def add_one(x): - raise ValueError("foo") - return x + 1 - - -def main(): - futures = [add_one.remote(i) for i in range(4)] - results = ray.get(futures) - assert results == [1, 2, 3, 4], f"Unexpected results: {results}" - - -if __name__ == "__main__": - main() - ray.shutdown() diff --git a/tests/contrib/ray/jobs/nested_tasks.py b/tests/contrib/ray/jobs/nested_tasks.py deleted file mode 100644 index df0fe7e2a30..00000000000 --- a/tests/contrib/ray/jobs/nested_tasks.py +++ /dev/null @@ -1,27 +0,0 @@ -import ray - - -ray.init() - - -@ray.remote -def add_one(x): - return x + 1 - - -@ray.remote -def submit_addition_task(x): - futures = [add_one.remote(x + i) for i in range(3)] - return ray.get(futures) - - -def main(): - future = submit_addition_task.remote(2) - - results = ray.get(future) - assert results == [3, 4, 5], f"Unexpected results: {results}" - - -if __name__ == "__main__": - main() - ray.shutdown() diff --git a/tests/contrib/ray/jobs/simple_actor.py b/tests/contrib/ray/jobs/simple_actor.py deleted file mode 100644 index ce2491cc3eb..00000000000 --- a/tests/contrib/ray/jobs/simple_actor.py +++ /dev/null @@ -1,37 +0,0 @@ -import ray - - -ray.init() - - -@ray.remote -class Counter: - def __init__(self): - self.value = 0 - - def increment(self): - self.value += 1 - return self.value - - def get_value(self): - return self.value - - def increment_and_get(self): - self.increment() - return self.get_value() - - def increment_get_and_double(self): - value = self.increment_and_get() - return value * 2 - - -def main(): - counter_actor = Counter.remote() - current_value = ray.get(counter_actor.increment_get_and_double.remote()) - - assert current_value == 2, f"Unexpected result: {current_value}" - - -if __name__ == "__main__": - main() - ray.shutdown() diff --git a/tests/contrib/ray/jobs/simple_task.py b/tests/contrib/ray/jobs/simple_task.py deleted file mode 100644 index 4d0bacf31ca..00000000000 --- a/tests/contrib/ray/jobs/simple_task.py +++ /dev/null @@ -1,20 +0,0 @@ -import ray - - -ray.init() - - -@ray.remote -def add_one(x): - return x + 1 - - -def main(): - futures = [add_one.remote(i) for i in range(4)] - results = ray.get(futures) - assert results == [1, 2, 3, 4], f"Unexpected results: {results}" - - -if __name__ == "__main__": - main() - ray.shutdown() diff --git a/tests/contrib/ray/jobs/simple_wait.py b/tests/contrib/ray/jobs/simple_wait.py deleted file mode 100644 index f3b5673a697..00000000000 --- a/tests/contrib/ray/jobs/simple_wait.py +++ /dev/null @@ -1,20 +0,0 @@ -import ray - - -ray.init() - - -@ray.remote -def add_one(x): - return x + 1 - - -def main(): - done, running = ray.wait([add_one.remote(42)], num_returns=1, timeout=60) - assert running == [], f"Expected no running tasks, got {len(running)}" - assert ray.get(done) == [43], f"Expected done to be [43], got {done}" - - -if __name__ == "__main__": - main() - ray.shutdown() diff --git a/tests/contrib/ray/jobs/task_without_init.py b/tests/contrib/ray/jobs/task_without_init.py new file mode 100644 index 00000000000..50e85e8e8a9 --- /dev/null +++ b/tests/contrib/ray/jobs/task_without_init.py @@ -0,0 +1,25 @@ +import ray + + +@ray.remote +def add_one(x): + return x + 1 + + +@ray.remote +def add_two(x): + return x + 2 + + +def main(): + futures_add_one = [add_one.remote(i) for i in range(2)] + results = ray.get(futures_add_one) + assert results == [1, 2], f"Unexpected results: {results}" + + futures_add_two = [add_two.remote(i) for i in range(2)] + results = ray.get(futures_add_two) + assert results == [2, 3], f"Unexpected results: {results}" + + +if __name__ == "__main__": + main() diff --git a/tests/contrib/ray/test_ray.py b/tests/contrib/ray/test_ray.py index 8f999ace5eb..e6be7a09488 100644 --- a/tests/contrib/ray/test_ray.py +++ b/tests/contrib/ray/test_ray.py @@ -1,3 +1,7 @@ +import os +import subprocess +import time + import pytest import ray from ray.util.tracing import tracing_helper @@ -314,19 +318,78 @@ def increment(self, x, y=1): class TestRayWithoutInit(TracerTestCase): - def tearDown(self): + """This class tests that actor and task submissions work with auto initialization + This class also shows a lack of visibility when we cannot call ray.init explictly + which led to a change in how we start a cluster + """ + + dashboard_url = None + + @classmethod + def setUpClass(cls): + super().setUpClass() + + env = os.environ.copy() + env.update( + { + "DD_PATCH_MODULES": "ray:true,aiohttp:false,grpc:false,requests:false", + } + ) + + subprocess.run( + [ + "ray", + "start", + "--head", + "--tracing-startup-hook=ddtrace.contrib.ray:setup_tracing", + "--num-cpus=1", + "--num-gpus=0", + "--object-store-memory=100000000", + "--dashboard-host=127.0.0.1", + "--dashboard-port=8265", + ], + env=env, + check=True, + ) + tracing_helper._global_is_tracing_enabled = False + + # Wait for dashboard to be ready + cls.dashboard_url = "http://127.0.0.1:8265" + time.sleep(2) + + @classmethod + def tearDownClass(cls): if ray.is_initialized(): ray.shutdown() - super().tearDown() + super().tearDownClass() + + def _submit_and_wait_for_job(self, job_script_name, timeout=30): + job_script = os.path.join(os.path.dirname(__file__), "jobs", job_script_name) + result = subprocess.run( + [ + "ray", + "job", + "submit", + "--address", + str(self.dashboard_url), + "--", + "python", + job_script, + ], + capture_output=True, + text=True, + timeout=timeout, + ) - @pytest.mark.snapshot(token="tests.contrib.ray.test_ray.test_task_without_init", ignores=RAY_SNAPSHOT_IGNORES) - def test_task_without_init(self): - """Test that tracing works when Ray auto-initializes without explicit ray.init()""" + assert result.returncode == 0, ( + f"Job failed with return code {result.returncode}. Stdout:\n{result.stdout}\nStderr:\n{result.stderr}" + ) + return result.stdout, result.stderr - @ray.remote - def add_one(x): - return x + 1 + @pytest.mark.snapshot(ignores=RAY_SNAPSHOT_IGNORES) + def test_task_without_init(self): + self._submit_and_wait_for_job("task_without_init.py") - futures = [add_one.remote(i) for i in range(2)] - results = ray.get(futures) - assert results == [1, 2], f"Unexpected results: {results}" + @pytest.mark.snapshot(ignores=RAY_SNAPSHOT_IGNORES) + def test_actor_without_init(self): + self._submit_and_wait_for_job("actor_without_init.py") diff --git a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json new file mode 100644 index 00000000000..ce0eadbe7c6 --- /dev/null +++ b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json @@ -0,0 +1,227 @@ +[[ + { + "name": "ray.job", + "service": "ray.dashboard", + "resource": "ray.job", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "ray", + "error": 0, + "meta": { + "_dd.p.dm": "-0", + "_dd.p.tid": "69286e5100000000", + "component": "ray", + "language": "python", + "ray.entrypoint": "python /actor_without_init.py", + "ray.hostname": "docker-desktop", + "ray.job.message": "Job finished successfully.", + "ray.job.status": "SUCCEEDED", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "dceb303a71f749c4b7a1e5bac8d8ad8d" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1470.0 + }, + "duration": 2822805210, + "start": 1764257361791674544 + }, + { + "name": "ray.job.submit", + "service": "ray.dashboard", + "resource": "ray.job.submit", + "trace_id": 0, + "span_id": 2, + "parent_id": 1, + "type": "ray", + "error": 0, + "meta": { + "component": "ray", + "ray.hostname": "docker-desktop", + "ray.job.submit_status": "success", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "span.kind": "producer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 76750, + "start": 1764257361791883669 + }, + { + "name": "actor_method.submit", + "service": "ray.dashboard", + "resource": "JobSupervisor.run.remote", + "trace_id": 0, + "span_id": 3, + "parent_id": 1, + "type": "ray", + "error": 0, + "meta": { + "component": "ray", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "span.kind": "producer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 231750, + "start": 1764257361824972877 + }, + { + "name": "actor_method.execute", + "service": "ray.dashboard", + "resource": "JobSupervisor.run", + "trace_id": 0, + "span_id": 4, + "parent_id": 3, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "_dd.p.tid": "69286e5100000000", + "_dd.parent_id": "b56189ba00c4fba5", + "component": "ray", + "language": "python", + "ray.actor_id": "542c7998936f4d95a7d3b82b01000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", + "ray.task_id": "16310a0f0a45af5c542c7998936f4d95a7d3b82b01000000", + "ray.worker_id": "651033a3df4aa187d150d080148a4a0dab23f4631828b7dfe72be235", + "runtime-id": "20b715d3bf544dcb9b6fa84adca78920", + "span.kind": "consumer", + "traceparent": "00-69286e5100000000cdefdffe2253a2bc-b56189ba00c4fba5-00", + "tracestate": "dd=p:b56189ba00c4fba5" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1648.0 + }, + "duration": 1929099251, + "start": 1764257362605382044 + }, + { + "name": "actor_method.execute", + "service": "ray.dashboard", + "resource": "JobSupervisor._get_driver_env_vars", + "trace_id": 0, + "span_id": 5, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "component": "ray", + "ray.actor_id": "542c7998936f4d95a7d3b82b01000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", + "ray.task_id": "16310a0f0a45af5c542c7998936f4d95a7d3b82b01000000", + "ray.worker_id": "651033a3df4aa187d150d080148a4a0dab23f4631828b7dfe72be235", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 1167042, + "start": 1764257362606739919 + }, + { + "name": "actor_method.execute", + "service": "ray.dashboard", + "resource": "JobSupervisor._get_driver_runtime_env", + "trace_id": 0, + "span_id": 7, + "parent_id": 5, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "component": "ray", + "ray.actor_id": "542c7998936f4d95a7d3b82b01000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", + "ray.task_id": "16310a0f0a45af5c542c7998936f4d95a7d3b82b01000000", + "ray.worker_id": "651033a3df4aa187d150d080148a4a0dab23f4631828b7dfe72be235", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 45166, + "start": 1764257362607837753 + }, + { + "name": "exec entrypoint", + "service": "ray.dashboard", + "resource": "exec actor_without_init.py", + "trace_id": 0, + "span_id": 6, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "component": "ray", + "ray.actor_id": "542c7998936f4d95a7d3b82b01000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", + "ray.task_id": "16310a0f0a45af5c542c7998936f4d95a7d3b82b01000000", + "ray.worker_id": "651033a3df4aa187d150d080148a4a0dab23f4631828b7dfe72be235", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 423000, + "start": 1764257362608085044 + }]] diff --git a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json new file mode 100644 index 00000000000..976fad6ac28 --- /dev/null +++ b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json @@ -0,0 +1,424 @@ +[[ + { + "name": "ray.job", + "service": "ray.dashboard", + "resource": "ray.job", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "ray", + "error": 0, + "meta": { + "_dd.p.dm": "-0", + "_dd.p.tid": "69286e5800000000", + "component": "ray", + "language": "python", + "ray.entrypoint": "python /task_without_init.py", + "ray.hostname": "docker-desktop", + "ray.job.message": "Job finished successfully.", + "ray.job.status": "SUCCEEDED", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "dceb303a71f749c4b7a1e5bac8d8ad8d" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1470.0 + }, + "duration": 2851458251, + "start": 1764257368399239214 + }, + { + "name": "ray.job.submit", + "service": "ray.dashboard", + "resource": "ray.job.submit", + "trace_id": 0, + "span_id": 2, + "parent_id": 1, + "type": "ray", + "error": 0, + "meta": { + "component": "ray", + "ray.hostname": "docker-desktop", + "ray.job.submit_status": "success", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "span.kind": "producer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 69333, + "start": 1764257368399504589 + }, + { + "name": "actor_method.submit", + "service": "ray.dashboard", + "resource": "JobSupervisor.run.remote", + "trace_id": 0, + "span_id": 3, + "parent_id": 1, + "type": "ray", + "error": 0, + "meta": { + "component": "ray", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "span.kind": "producer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 403083, + "start": 1764257368403539422 + }, + { + "name": "actor_method.execute", + "service": "ray.dashboard", + "resource": "JobSupervisor.run", + "trace_id": 0, + "span_id": 4, + "parent_id": 3, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "_dd.p.tid": "69286e5800000000", + "_dd.parent_id": "339673445e84f782", + "component": "ray", + "language": "python", + "ray.actor_id": "611d7d8b4b40eb8362cca23301000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.task_id": "f91b78d7db9a6593611d7d8b4b40eb8362cca23301000000", + "ray.worker_id": "9a4dda773aba8c8fe80ba86b4ef71e66564f81bcb25edd2900f3a643", + "runtime-id": "e55cc8e871d3448595d33efc029af605", + "span.kind": "consumer", + "traceparent": "00-69286e5800000000abceee630a6a000f-339673445e84f782-00", + "tracestate": "dd=p:339673445e84f782" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1835.0 + }, + "duration": 1933514251, + "start": 1764257369240291547 + }, + { + "name": "actor_method.execute", + "service": "ray.dashboard", + "resource": "JobSupervisor._get_driver_env_vars", + "trace_id": 0, + "span_id": 5, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "component": "ray", + "ray.actor_id": "611d7d8b4b40eb8362cca23301000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.task_id": "f91b78d7db9a6593611d7d8b4b40eb8362cca23301000000", + "ray.worker_id": "9a4dda773aba8c8fe80ba86b4ef71e66564f81bcb25edd2900f3a643", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 1505625, + "start": 1764257369242125547 + }, + { + "name": "actor_method.execute", + "service": "ray.dashboard", + "resource": "JobSupervisor._get_driver_runtime_env", + "trace_id": 0, + "span_id": 10, + "parent_id": 5, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "component": "ray", + "ray.actor_id": "611d7d8b4b40eb8362cca23301000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.task_id": "f91b78d7db9a6593611d7d8b4b40eb8362cca23301000000", + "ray.worker_id": "9a4dda773aba8c8fe80ba86b4ef71e66564f81bcb25edd2900f3a643", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 63291, + "start": 1764257369243536881 + }, + { + "name": "exec entrypoint", + "service": "ray.dashboard", + "resource": "exec task_without_init.py", + "trace_id": 0, + "span_id": 6, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "component": "ray", + "ray.actor_id": "611d7d8b4b40eb8362cca23301000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.task_id": "f91b78d7db9a6593611d7d8b4b40eb8362cca23301000000", + "ray.worker_id": "9a4dda773aba8c8fe80ba86b4ef71e66564f81bcb25edd2900f3a643", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 482000, + "start": 1764257369243832256 + }, + { + "name": "task.submit", + "service": "ray.dashboard", + "resource": "__main__.add_one.remote", + "trace_id": 0, + "span_id": 7, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "project", + "_dd.p.tid": "69286e5800000000", + "_dd.parent_id": "2cd921d88a156042", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "03000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.task.submit_status": "success", + "ray.worker_id": "03000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "e6f91ed4f46641bb93e5625bb348d5d6", + "span.kind": "producer", + "traceparent": "00-69286e5800000000abceee630a6a000f-2cd921d88a156042-00", + "tracestate": "dd=p:2cd921d88a156042;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1888.0 + }, + "duration": 232583, + "start": 1764257369740446173 + }, + { + "name": "task.submit", + "service": "ray.dashboard", + "resource": "__main__.add_two.remote", + "trace_id": 0, + "span_id": 8, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "project", + "_dd.p.tid": "69286e5800000000", + "_dd.parent_id": "2cd921d88a156042", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "03000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.task.submit_status": "success", + "ray.worker_id": "03000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "e6f91ed4f46641bb93e5625bb348d5d6", + "span.kind": "producer", + "traceparent": "00-69286e5800000000abceee630a6a000f-2cd921d88a156042-00", + "tracestate": "dd=p:2cd921d88a156042;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1888.0 + }, + "duration": 1459625, + "start": 1764257370473452465 + }, + { + "name": "task.execute", + "service": "ray.dashboard", + "resource": "__main__.add_two", + "trace_id": 0, + "span_id": 11, + "parent_id": 8, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "_dd.p.tid": "69286e5800000000", + "_dd.parent_id": "8d237abcbfe86ed1", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "03000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.task.status": "success", + "ray.task_id": "0ab01f2d6283d719ffffffffffffffffffffffff03000000", + "ray.worker_id": "ee8139d01a3b0958ba02bebad045b206719d8bd9d3fc6193768e57e6", + "runtime-id": "4c250db564d64507a00fe48a38ce5996", + "span.kind": "consumer", + "traceparent": "00-69286e5800000000abceee630a6a000f-8d237abcbfe86ed1-00", + "tracestate": "dd=p:8d237abcbfe86ed1;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1936.0 + }, + "duration": 273667, + "start": 1764257370476360006 + }, + { + "name": "task.submit", + "service": "ray.dashboard", + "resource": "__main__.add_two.remote", + "trace_id": 0, + "span_id": 9, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "project", + "_dd.p.tid": "69286e5800000000", + "_dd.parent_id": "2cd921d88a156042", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "03000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.task.submit_status": "success", + "ray.worker_id": "03000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "e6f91ed4f46641bb93e5625bb348d5d6", + "span.kind": "producer", + "traceparent": "00-69286e5800000000abceee630a6a000f-2cd921d88a156042-00", + "tracestate": "dd=p:2cd921d88a156042;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1888.0 + }, + "duration": 225959, + "start": 1764257370475828131 + }, + { + "name": "task.execute", + "service": "ray.dashboard", + "resource": "__main__.add_two", + "trace_id": 0, + "span_id": 12, + "parent_id": 9, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "_dd.p.tid": "69286e5800000000", + "_dd.parent_id": "25ff86ae80db82af", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "03000000", + "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.task.status": "success", + "ray.task_id": "62ffec03f52574bdffffffffffffffffffffffff03000000", + "ray.worker_id": "ee8139d01a3b0958ba02bebad045b206719d8bd9d3fc6193768e57e6", + "runtime-id": "4c250db564d64507a00fe48a38ce5996", + "span.kind": "consumer", + "traceparent": "00-69286e5800000000abceee630a6a000f-25ff86ae80db82af-00", + "tracestate": "dd=p:25ff86ae80db82af;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1936.0 + }, + "duration": 138750, + "start": 1764257370478251923 + }]] From 8683e2e87e705f88a361eccb52a1dad3df49db31 Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Fri, 28 Nov 2025 10:03:30 +0100 Subject: [PATCH 2/8] fix tests --- tests/contrib/ray/test_ray.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/contrib/ray/test_ray.py b/tests/contrib/ray/test_ray.py index e6be7a09488..a82178472d0 100644 --- a/tests/contrib/ray/test_ray.py +++ b/tests/contrib/ray/test_ray.py @@ -359,8 +359,15 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - if ray.is_initialized(): - ray.shutdown() + try: + subprocess.run( + ["ray", "stop"], + check=False, + capture_output=True, + ) + except Exception: + pass # ignore cleanup errors + super().tearDownClass() def _submit_and_wait_for_job(self, job_script_name, timeout=30): From bb79370b112b20b9cded826149513dfdb005113a Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Fri, 28 Nov 2025 11:49:45 +0100 Subject: [PATCH 3/8] fix service names --- ddtrace/contrib/internal/ray/__init__.py | 12 +- ddtrace/contrib/internal/ray/patch.py | 24 +- tests/contrib/ray/jobs/service.py | 16 + tests/contrib/ray/test_ray.py | 169 +++++--- ...ayWithoutInit.test_actor_without_init.json | 196 ++++++--- ...ayWithoutInit.test_job_name_specified.json | 307 ++++++++++++++ ...RayWithoutInit.test_task_without_init.json | 374 ++++++++++++------ ...ontrib.ray.test_ray.test_service_name.json | 298 ++++++++++++++ ...b.ray.test_ray.test_task_without_init.json | 69 ---- ..._ray.test_use_entrypoint_service_name.json | 307 ++++++++++++++ 10 files changed, 1435 insertions(+), 337 deletions(-) create mode 100644 tests/contrib/ray/jobs/service.py create mode 100644 tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_job_name_specified.json create mode 100644 tests/snapshots/tests.contrib.ray.test_ray.test_service_name.json delete mode 100644 tests/snapshots/tests.contrib.ray.test_ray.test_task_without_init.json create mode 100644 tests/snapshots/tests.contrib.ray.test_ray.test_use_entrypoint_service_name.json diff --git a/ddtrace/contrib/internal/ray/__init__.py b/ddtrace/contrib/internal/ray/__init__.py index b2499ded631..33e6ce1ca00 100644 --- a/ddtrace/contrib/internal/ray/__init__.py +++ b/ddtrace/contrib/internal/ray/__init__.py @@ -47,21 +47,17 @@ (default: ``True``). If ``True``, file paths in the entrypoint will be redacted to avoid leaking sensitive information. -Ray service name can be configured by: +Ray service name can be configured, in order of precedence by: -- specifying in submission ID using ``job:your-job-name`` during job submission:: +- specifying ``DD_SERVICE`` when initializing your Ray cluster. - ray job submit --submission-id="job:my_model,run:39" -- python entrypoint.py +- setting ``DD_TRACE_RAY_USE_ENTRYPOINT_AS_SERVICE_NAME=True``. In this case, the service + name will be the name of your entrypoint script. - specifying in metadata during job submission:: ray job submit --metadata-json='{"job_name": "my_model"}' -- python entrypoint.py -- specifying ``DD_SERVICE`` when initializing your Ray cluster. - -- setting ``DD_TRACE_RAY_USE_ENTRYPOINT_AS_SERVICE_NAME=True``. In this case, the service - name will be the name of your entrypoint script. - By default, the service name will be ``unnamed.ray.job``. Notes diff --git a/ddtrace/contrib/internal/ray/patch.py b/ddtrace/contrib/internal/ray/patch.py index 9957944f390..f0196e22e60 100644 --- a/ddtrace/contrib/internal/ray/patch.py +++ b/ddtrace/contrib/internal/ray/patch.py @@ -196,22 +196,24 @@ def traced_submit_job(wrapped, instance, args, kwargs): """ from ray.dashboard.modules.job.job_manager import generate_job_id - # Three ways of specifying the job name, in order of precedence: - # 1. Metadata JSON: ray job submit --metadata_json '{"job_name": "train.cool.model"}' train.py - # 2. Special submission ID format: ray job submit --submission_id "job:train.cool.model,run:38" train.py - # 3. Ray entrypoint: ray job submit train_cool_model.py + # Three ways of setting the service name of the spans, in order of precedence: + # - DD_SERVICE environment variable + # - The name of the entrypoint if DD_TRACE_RAY_USE_ENTRYPOINT_AS_SERVICE_NAME is True + # - Metadata JSON: ray job submit --metadata_json '{"job_name": "train.cool.model"}' + # Otherwise set to unnamed.ray.job submission_id = kwargs.get("submission_id") or generate_job_id() kwargs["submission_id"] = submission_id + entrypoint = kwargs.get("entrypoint", "") - if entrypoint and config.ray.redact_entrypoint_paths: + if config.ray.redact_entrypoint_paths: entrypoint = redact_paths(entrypoint) - job_name = config.service or kwargs.get("metadata", {}).get("job_name", "") - if not job_name: - if config.ray.use_entrypoint_as_service_name: - job_name = get_dd_job_name_from_entrypoint(entrypoint) or DEFAULT_JOB_NAME - else: - job_name = DEFAULT_JOB_NAME + if config.ray.use_entrypoint_as_service_name: + job_name = get_dd_job_name_from_entrypoint(entrypoint) or DEFAULT_JOB_NAME + else: + user_provided_service = config.service if config._is_user_provided_service else None + metadata_job_name = kwargs.get("metadata", {}).get("job_name", None) + job_name = user_provided_service or metadata_job_name or DEFAULT_JOB_NAME job_span = tracer.start_span("ray.job", service=job_name or DEFAULT_JOB_NAME, span_type=SpanTypes.RAY) try: diff --git a/tests/contrib/ray/jobs/service.py b/tests/contrib/ray/jobs/service.py new file mode 100644 index 00000000000..053324873c2 --- /dev/null +++ b/tests/contrib/ray/jobs/service.py @@ -0,0 +1,16 @@ +import ray + + +@ray.remote +def add_one(x): + return x + 1 + + +def main(): + futures_add_one = add_one.remote(0) + result = ray.get(futures_add_one) + assert result == 1, f"Unexpected results: {result}" + + +if __name__ == "__main__": + main() diff --git a/tests/contrib/ray/test_ray.py b/tests/contrib/ray/test_ray.py index a82178472d0..c3834b0f6c9 100644 --- a/tests/contrib/ray/test_ray.py +++ b/tests/contrib/ray/test_ray.py @@ -1,3 +1,4 @@ +import json import os import subprocess import time @@ -47,13 +48,10 @@ def ray_cluster(self): ray.init( _tracing_startup_hook="ddtrace.contrib.ray:setup_tracing", local_mode=True, - # Limit resources to reduce CI load num_cpus=1, num_gpus=0, object_store_memory=78643200, - # Disable dashboard to save memory include_dashboard=False, - # Set log level to reduce I/O log_to_driver=False, ) tracing_helper._global_is_tracing_enabled = False @@ -317,6 +315,74 @@ def increment(self, x, y=1): assert current_value == 3, f"Unexpected result: {current_value}" +def _start_ray_cluster(env_vars=None): + """Start a Ray cluster with optional environment variables.""" + env = os.environ.copy() + base_env = { + "DD_PATCH_MODULES": "ray:true,aiohttp:false,grpc:false,requests:false", + } + if env_vars: + base_env.update(env_vars) + env.update(base_env) + + subprocess.run( + [ + "ddtrace-run", + "ray", + "start", + "--head", + "--num-cpus=1", + "--num-gpus=0", + "--object-store-memory=100000000", + "--dashboard-host=127.0.0.1", + "--dashboard-port=8265", + ], + env=env, + check=True, + ) + # Wait for dashboard to be ready + time.sleep(2) + return "http://127.0.0.1:8265" + + +def _stop_ray_cluster(): + """Stop the Ray cluster, ignoring any errors.""" + try: + subprocess.run( + ["ray", "stop"], + check=False, + capture_output=True, + ) + except Exception: + pass # ignore cleanup errors + + +def _submit_and_wait_for_job(dashboard_url, job_script_name, metadata={"foo": "bar"}, timeout=30): + """Submit a Ray job and wait for completion.""" + job_script = os.path.join(os.path.dirname(__file__), "jobs", job_script_name) + result = subprocess.run( + [ + "ray", + "job", + "submit", + f"--metadata-json={json.dumps(metadata)}", + "--address", + str(dashboard_url), + "--", + "python", + job_script, + ], + capture_output=True, + text=True, + timeout=timeout, + ) + + assert result.returncode == 0, ( + f"Job failed with return code {result.returncode}. Stdout:\n{result.stdout}\nStderr:\n{result.stderr}" + ) + return result.stdout, result.stderr + + class TestRayWithoutInit(TracerTestCase): """This class tests that actor and task submissions work with auto initialization This class also shows a lack of visibility when we cannot call ray.init explictly @@ -328,75 +394,48 @@ class TestRayWithoutInit(TracerTestCase): @classmethod def setUpClass(cls): super().setUpClass() - - env = os.environ.copy() - env.update( - { - "DD_PATCH_MODULES": "ray:true,aiohttp:false,grpc:false,requests:false", - } - ) - - subprocess.run( - [ - "ray", - "start", - "--head", - "--tracing-startup-hook=ddtrace.contrib.ray:setup_tracing", - "--num-cpus=1", - "--num-gpus=0", - "--object-store-memory=100000000", - "--dashboard-host=127.0.0.1", - "--dashboard-port=8265", - ], - env=env, - check=True, - ) - tracing_helper._global_is_tracing_enabled = False - - # Wait for dashboard to be ready - cls.dashboard_url = "http://127.0.0.1:8265" - time.sleep(2) + cls.dashboard_url = _start_ray_cluster() @classmethod def tearDownClass(cls): - try: - subprocess.run( - ["ray", "stop"], - check=False, - capture_output=True, - ) - except Exception: - pass # ignore cleanup errors - + _stop_ray_cluster() super().tearDownClass() - def _submit_and_wait_for_job(self, job_script_name, timeout=30): - job_script = os.path.join(os.path.dirname(__file__), "jobs", job_script_name) - result = subprocess.run( - [ - "ray", - "job", - "submit", - "--address", - str(self.dashboard_url), - "--", - "python", - job_script, - ], - capture_output=True, - text=True, - timeout=timeout, - ) - - assert result.returncode == 0, ( - f"Job failed with return code {result.returncode}. Stdout:\n{result.stdout}\nStderr:\n{result.stderr}" - ) - return result.stdout, result.stderr - @pytest.mark.snapshot(ignores=RAY_SNAPSHOT_IGNORES) def test_task_without_init(self): - self._submit_and_wait_for_job("task_without_init.py") + _submit_and_wait_for_job(self.dashboard_url, "task_without_init.py") @pytest.mark.snapshot(ignores=RAY_SNAPSHOT_IGNORES) def test_actor_without_init(self): - self._submit_and_wait_for_job("actor_without_init.py") + _submit_and_wait_for_job(self.dashboard_url, "actor_without_init.py") + + @pytest.mark.snapshot(ignores=RAY_SNAPSHOT_IGNORES) + def test_job_name_specified(self): + _submit_and_wait_for_job(self.dashboard_url, "service.py", metadata={"job_name": "my_model"}) + + +@pytest.mark.snapshot(ignores=RAY_SNAPSHOT_IGNORES) +def test_service_name(): + """Test that DD_SERVICE environment variable is used as service name.""" + dashboard_url = _start_ray_cluster(env_vars={"DD_SERVICE": "test"}) + + try: + _submit_and_wait_for_job(dashboard_url, "service.py") + finally: + _stop_ray_cluster() + + +@pytest.mark.snapshot(ignores=RAY_SNAPSHOT_IGNORES) +def test_use_entrypoint_service_name(): + """Test that entrypoint can be used as service name when configured.""" + dashboard_url = _start_ray_cluster( + env_vars={ + "DD_SERVICE": "test", + "DD_TRACE_RAY_USE_ENTRYPOINT_AS_SERVICE_NAME": "True", + } + ) + + try: + _submit_and_wait_for_job(dashboard_url, "service.py") + finally: + _stop_ray_cluster() diff --git a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json index ce0eadbe7c6..8fb94faf027 100644 --- a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json +++ b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json @@ -1,7 +1,7 @@ [[ { "name": "ray.job", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "ray.job", "trace_id": 0, "span_id": 1, @@ -9,19 +9,21 @@ "type": "ray", "error": 0, "meta": { + "_dd.base_service": "ray.dashboard", "_dd.p.dm": "-0", - "_dd.p.tid": "69286e5100000000", + "_dd.p.tid": "69297d9000000000", "component": "ray", "language": "python", "ray.entrypoint": "python /actor_without_init.py", "ray.hostname": "docker-desktop", "ray.job.message": "Job finished successfully.", + "ray.job.metadata.foo": "bar", "ray.job.status": "SUCCEEDED", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "dceb303a71f749c4b7a1e5bac8d8ad8d" + "runtime-id": "81f567481c4e41dc8f1b56ecf05d2973" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -31,14 +33,14 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1470.0 + "process_id": 1599.0 }, - "duration": 2822805210, - "start": 1764257361791674544 + "duration": 4046849919, + "start": 1764326800832845842 }, { "name": "ray.job.submit", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "ray.job.submit", "trace_id": 0, "span_id": 2, @@ -46,12 +48,13 @@ "type": "ray", "error": 0, "meta": { + "_dd.base_service": "ray.dashboard", "component": "ray", "ray.hostname": "docker-desktop", "ray.job.submit_status": "success", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", "span.kind": "producer" }, @@ -62,12 +65,12 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 76750, - "start": 1764257361791883669 + "duration": 59333, + "start": 1764326800833023842 }, { "name": "actor_method.submit", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "JobSupervisor.run.remote", "trace_id": 0, "span_id": 3, @@ -75,10 +78,11 @@ "type": "ray", "error": 0, "meta": { + "_dd.base_service": "ray.dashboard", "component": "ray", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", "span.kind": "producer" }, @@ -89,12 +93,12 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 231750, - "start": 1764257361824972877 + "duration": 264542, + "start": 1764326800873390467 }, { "name": "actor_method.execute", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "JobSupervisor.run", "trace_id": 0, "span_id": 4, @@ -103,21 +107,21 @@ "error": 0, "meta": { "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69286e5100000000", - "_dd.parent_id": "b56189ba00c4fba5", + "_dd.p.tid": "69297d9000000000", + "_dd.parent_id": "b8e28cc5ec16e6c9", "component": "ray", "language": "python", - "ray.actor_id": "542c7998936f4d95a7d3b82b01000000", + "ray.actor_id": "5d7b8f890a255711f2ceb2b401000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", - "ray.task_id": "16310a0f0a45af5c542c7998936f4d95a7d3b82b01000000", - "ray.worker_id": "651033a3df4aa187d150d080148a4a0dab23f4631828b7dfe72be235", - "runtime-id": "20b715d3bf544dcb9b6fa84adca78920", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", + "ray.task_id": "16310a0f0a45af5c5d7b8f890a255711f2ceb2b401000000", + "ray.worker_id": "08b10fdcbdb3706b74070824f41e97ce7ce7fb7f98b5ab24b6c69951", + "runtime-id": "9bd2dd0007f84438b2e83a857940d05d", "span.kind": "consumer", - "traceparent": "00-69286e5100000000cdefdffe2253a2bc-b56189ba00c4fba5-00", - "tracestate": "dd=p:b56189ba00c4fba5" + "traceparent": "00-69297d90000000007b46d2532fc2b819-b8e28cc5ec16e6c9-00", + "tracestate": "dd=p:b8e28cc5ec16e6c9" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -127,14 +131,14 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1648.0 + "process_id": 1787.0 }, - "duration": 1929099251, - "start": 1764257362605382044 + "duration": 2031085667, + "start": 1764326801860054676 }, { "name": "actor_method.execute", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "JobSupervisor._get_driver_env_vars", "trace_id": 0, "span_id": 5, @@ -144,13 +148,13 @@ "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "542c7998936f4d95a7d3b82b01000000", + "ray.actor_id": "5d7b8f890a255711f2ceb2b401000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", - "ray.task_id": "16310a0f0a45af5c542c7998936f4d95a7d3b82b01000000", - "ray.worker_id": "651033a3df4aa187d150d080148a4a0dab23f4631828b7dfe72be235", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", + "ray.task_id": "16310a0f0a45af5c5d7b8f890a255711f2ceb2b401000000", + "ray.worker_id": "08b10fdcbdb3706b74070824f41e97ce7ce7fb7f98b5ab24b6c69951", "span.kind": "consumer" }, "metrics": { @@ -160,28 +164,28 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 1167042, - "start": 1764257362606739919 + "duration": 1300375, + "start": 1764326801861597967 }, { "name": "actor_method.execute", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "JobSupervisor._get_driver_runtime_env", "trace_id": 0, - "span_id": 7, + "span_id": 8, "parent_id": 5, "type": "ray", "error": 0, "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "542c7998936f4d95a7d3b82b01000000", + "ray.actor_id": "5d7b8f890a255711f2ceb2b401000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", - "ray.task_id": "16310a0f0a45af5c542c7998936f4d95a7d3b82b01000000", - "ray.worker_id": "651033a3df4aa187d150d080148a4a0dab23f4631828b7dfe72be235", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", + "ray.task_id": "16310a0f0a45af5c5d7b8f890a255711f2ceb2b401000000", + "ray.worker_id": "08b10fdcbdb3706b74070824f41e97ce7ce7fb7f98b5ab24b6c69951", "span.kind": "consumer" }, "metrics": { @@ -191,12 +195,12 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 45166, - "start": 1764257362607837753 + "duration": 49042, + "start": 1764326801862827092 }, { "name": "exec entrypoint", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "exec actor_without_init.py", "trace_id": 0, "span_id": 6, @@ -206,13 +210,13 @@ "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "542c7998936f4d95a7d3b82b01000000", + "ray.actor_id": "5d7b8f890a255711f2ceb2b401000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_jauRVvbAgLsYYtg5", - "ray.task_id": "16310a0f0a45af5c542c7998936f4d95a7d3b82b01000000", - "ray.worker_id": "651033a3df4aa187d150d080148a4a0dab23f4631828b7dfe72be235", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", + "ray.task_id": "16310a0f0a45af5c5d7b8f890a255711f2ceb2b401000000", + "ray.worker_id": "08b10fdcbdb3706b74070824f41e97ce7ce7fb7f98b5ab24b6c69951", "span.kind": "consumer" }, "metrics": { @@ -222,6 +226,84 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 423000, - "start": 1764257362608085044 - }]] + "duration": 448708, + "start": 1764326801863071426 + }, + { + "name": "actor_method.submit", + "service": "unnamed.ray.job", + "resource": "Counter.increment.remote", + "trace_id": 0, + "span_id": 7, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "project", + "_dd.p.tid": "69297d9000000000", + "_dd.parent_id": "1ef55996279e487b", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "02000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", + "ray.worker_id": "02000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "1650ca0c39a544418fd27eaba368aa5e", + "span.kind": "producer", + "traceparent": "00-69297d90000000007b46d2532fc2b819-1ef55996279e487b-00", + "tracestate": "dd=p:1ef55996279e487b;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1844.0 + }, + "duration": 301292, + "start": 1764326802310894009 + }, + { + "name": "actor_method.execute", + "service": "unnamed.ray.job", + "resource": "Counter.increment", + "trace_id": 0, + "span_id": 9, + "parent_id": 7, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "_dd.p.tid": "69297d9000000000", + "_dd.parent_id": "40bcf0c7bb993b59", + "component": "ray", + "language": "python", + "ray.actor_id": "b9b2522c73101eb8596ed95a02000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "02000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", + "ray.task_id": "e082c90ab8422b00b9b2522c73101eb8596ed95a02000000", + "ray.worker_id": "079fd52b7ed068a3f5c0913d3d3b740ca854d0ac958e4c213859c1d1", + "runtime-id": "be7f7861b5f5452e8d33426a3682f6da", + "span.kind": "consumer", + "traceparent": "00-69297d90000000007b46d2532fc2b819-40bcf0c7bb993b59-00", + "tracestate": "dd=p:40bcf0c7bb993b59;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1892.0 + }, + "duration": 197291, + "start": 1764326803137070760 + }]] diff --git a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_job_name_specified.json b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_job_name_specified.json new file mode 100644 index 00000000000..d3d072f5855 --- /dev/null +++ b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_job_name_specified.json @@ -0,0 +1,307 @@ +[[ + { + "name": "ray.job", + "service": "my_model", + "resource": "ray.job", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray.dashboard", + "_dd.p.dm": "-0", + "_dd.p.tid": "69297d9700000000", + "component": "ray", + "language": "python", + "ray.entrypoint": "python /service.py", + "ray.hostname": "docker-desktop", + "ray.job.message": "Job finished successfully.", + "ray.job.metadata.job_name": "my_model", + "ray.job.status": "SUCCEEDED", + "ray.job_id": "01000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "81f567481c4e41dc8f1b56ecf05d2973" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1599.0 + }, + "duration": 2987849876, + "start": 1764326807461016012 + }, + { + "name": "ray.job.submit", + "service": "my_model", + "resource": "ray.job.submit", + "trace_id": 0, + "span_id": 2, + "parent_id": 1, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray.dashboard", + "component": "ray", + "ray.hostname": "docker-desktop", + "ray.job.submit_status": "success", + "ray.job_id": "01000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "span.kind": "producer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 56250, + "start": 1764326807461274095 + }, + { + "name": "actor_method.submit", + "service": "my_model", + "resource": "JobSupervisor.run.remote", + "trace_id": 0, + "span_id": 3, + "parent_id": 1, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray.dashboard", + "component": "ray", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "span.kind": "producer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 779417, + "start": 1764326807466330845 + }, + { + "name": "actor_method.execute", + "service": "my_model", + "resource": "JobSupervisor.run", + "trace_id": 0, + "span_id": 4, + "parent_id": 3, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "_dd.p.tid": "69297d9700000000", + "_dd.parent_id": "ef4b527279a09cfc", + "component": "ray", + "language": "python", + "ray.actor_id": "188ec803f749feb4a24a286901000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", + "ray.task_id": "82891771158d68c1188ec803f749feb4a24a286901000000", + "ray.worker_id": "73fd7ee2c79bd5108091d730a28b51046985a81f470fdc733ff23bcc", + "runtime-id": "c209b08c9fa04906971e43a2489a7118", + "span.kind": "consumer", + "traceparent": "00-69297d970000000065054211638fe87d-ef4b527279a09cfc-00", + "tracestate": "dd=p:ef4b527279a09cfc" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 1982.0 + }, + "duration": 1929833376, + "start": 1764326808436916137 + }, + { + "name": "actor_method.execute", + "service": "my_model", + "resource": "JobSupervisor._get_driver_env_vars", + "trace_id": 0, + "span_id": 5, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "component": "ray", + "ray.actor_id": "188ec803f749feb4a24a286901000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", + "ray.task_id": "82891771158d68c1188ec803f749feb4a24a286901000000", + "ray.worker_id": "73fd7ee2c79bd5108091d730a28b51046985a81f470fdc733ff23bcc", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 1392625, + "start": 1764326808438404971 + }, + { + "name": "actor_method.execute", + "service": "my_model", + "resource": "JobSupervisor._get_driver_runtime_env", + "trace_id": 0, + "span_id": 8, + "parent_id": 5, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "component": "ray", + "ray.actor_id": "188ec803f749feb4a24a286901000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", + "ray.task_id": "82891771158d68c1188ec803f749feb4a24a286901000000", + "ray.worker_id": "73fd7ee2c79bd5108091d730a28b51046985a81f470fdc733ff23bcc", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 50750, + "start": 1764326808439718179 + }, + { + "name": "exec entrypoint", + "service": "my_model", + "resource": "exec service.py", + "trace_id": 0, + "span_id": 6, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "component": "ray", + "ray.actor_id": "188ec803f749feb4a24a286901000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", + "ray.task_id": "82891771158d68c1188ec803f749feb4a24a286901000000", + "ray.worker_id": "73fd7ee2c79bd5108091d730a28b51046985a81f470fdc733ff23bcc", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 417667, + "start": 1764326808439976179 + }, + { + "name": "task.submit", + "service": "my_model", + "resource": "__main__.add_one.remote", + "trace_id": 0, + "span_id": 7, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "project", + "_dd.p.tid": "69297d9700000000", + "_dd.parent_id": "e4ea4e5b0eb7e1f6", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", + "ray.task.submit_status": "success", + "runtime-id": "799d236cda0040bf9119c0ab230d0b43", + "span.kind": "producer", + "traceparent": "00-69297d970000000065054211638fe87d-e4ea4e5b0eb7e1f6-00", + "tracestate": "dd=p:e4ea4e5b0eb7e1f6;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 2039.0 + }, + "duration": 51639750, + "start": 1764326808835367429 + }, + { + "name": "task.execute", + "service": "my_model", + "resource": "__main__.add_one", + "trace_id": 0, + "span_id": 9, + "parent_id": 7, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "_dd.p.tid": "69297d9700000000", + "_dd.parent_id": "a39c9bc39cce2b3e", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "03000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", + "ray.task.status": "success", + "ray.task_id": "fa31b89f94899135ffffffffffffffffffffffff03000000", + "ray.worker_id": "d5b3492c212cb35721dab353aa08670ed0020a8b262672d6ab6dc482", + "runtime-id": "05f200c2a93e4003bb5204474396476d", + "span.kind": "consumer", + "traceparent": "00-69297d970000000065054211638fe87d-a39c9bc39cce2b3e-00", + "tracestate": "dd=p:a39c9bc39cce2b3e;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 2087.0 + }, + "duration": 217875, + "start": 1764326809733088096 + }]] diff --git a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json index 976fad6ac28..0e51247d8e2 100644 --- a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json +++ b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json @@ -1,7 +1,7 @@ [[ { "name": "ray.job", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "ray.job", "trace_id": 0, "span_id": 1, @@ -9,19 +9,21 @@ "type": "ray", "error": 0, "meta": { + "_dd.base_service": "ray.dashboard", "_dd.p.dm": "-0", - "_dd.p.tid": "69286e5800000000", + "_dd.p.tid": "69297d9d00000000", "component": "ray", "language": "python", "ray.entrypoint": "python /task_without_init.py", "ray.hostname": "docker-desktop", "ray.job.message": "Job finished successfully.", + "ray.job.metadata.foo": "bar", "ray.job.status": "SUCCEEDED", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "dceb303a71f749c4b7a1e5bac8d8ad8d" + "runtime-id": "81f567481c4e41dc8f1b56ecf05d2973" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -31,14 +33,14 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1470.0 + "process_id": 1599.0 }, - "duration": 2851458251, - "start": 1764257368399239214 + "duration": 3928049627, + "start": 1764326813998606834 }, { "name": "ray.job.submit", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "ray.job.submit", "trace_id": 0, "span_id": 2, @@ -46,12 +48,13 @@ "type": "ray", "error": 0, "meta": { + "_dd.base_service": "ray.dashboard", "component": "ray", "ray.hostname": "docker-desktop", "ray.job.submit_status": "success", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", "span.kind": "producer" }, @@ -62,12 +65,12 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 69333, - "start": 1764257368399504589 + "duration": 62292, + "start": 1764326813998850042 }, { "name": "actor_method.submit", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "JobSupervisor.run.remote", "trace_id": 0, "span_id": 3, @@ -75,10 +78,11 @@ "type": "ray", "error": 0, "meta": { + "_dd.base_service": "ray.dashboard", "component": "ray", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", "span.kind": "producer" }, @@ -89,12 +93,12 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 403083, - "start": 1764257368403539422 + "duration": 387917, + "start": 1764326814002223584 }, { "name": "actor_method.execute", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "JobSupervisor.run", "trace_id": 0, "span_id": 4, @@ -103,21 +107,21 @@ "error": 0, "meta": { "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69286e5800000000", - "_dd.parent_id": "339673445e84f782", + "_dd.p.tid": "69297d9d00000000", + "_dd.parent_id": "d90560ba034420ae", "component": "ray", "language": "python", - "ray.actor_id": "611d7d8b4b40eb8362cca23301000000", + "ray.actor_id": "690dcee46cf26e93ee89e59001000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", - "ray.task_id": "f91b78d7db9a6593611d7d8b4b40eb8362cca23301000000", - "ray.worker_id": "9a4dda773aba8c8fe80ba86b4ef71e66564f81bcb25edd2900f3a643", - "runtime-id": "e55cc8e871d3448595d33efc029af605", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.task_id": "85748392bcd969cc690dcee46cf26e93ee89e59001000000", + "ray.worker_id": "75bc236cdf71c7b5a1a4fa31b673d7f14572c4437fbc95fcedec2fa5", + "runtime-id": "c5a28ebae0c24da5b0a6f427ed8db095", "span.kind": "consumer", - "traceparent": "00-69286e5800000000abceee630a6a000f-339673445e84f782-00", - "tracestate": "dd=p:339673445e84f782" + "traceparent": "00-69297d9d00000000bf349262b273a9f9-d90560ba034420ae-00", + "tracestate": "dd=p:d90560ba034420ae" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -127,14 +131,14 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1835.0 + "process_id": 2153.0 }, - "duration": 1933514251, - "start": 1764257369240291547 + "duration": 2042603918, + "start": 1764326814908696334 }, { "name": "actor_method.execute", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "JobSupervisor._get_driver_env_vars", "trace_id": 0, "span_id": 5, @@ -144,13 +148,13 @@ "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "611d7d8b4b40eb8362cca23301000000", + "ray.actor_id": "690dcee46cf26e93ee89e59001000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", - "ray.task_id": "f91b78d7db9a6593611d7d8b4b40eb8362cca23301000000", - "ray.worker_id": "9a4dda773aba8c8fe80ba86b4ef71e66564f81bcb25edd2900f3a643", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.task_id": "85748392bcd969cc690dcee46cf26e93ee89e59001000000", + "ray.worker_id": "75bc236cdf71c7b5a1a4fa31b673d7f14572c4437fbc95fcedec2fa5", "span.kind": "consumer" }, "metrics": { @@ -160,28 +164,28 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 1505625, - "start": 1764257369242125547 + "duration": 1157500, + "start": 1764326814910158876 }, { "name": "actor_method.execute", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "JobSupervisor._get_driver_runtime_env", "trace_id": 0, - "span_id": 10, + "span_id": 11, "parent_id": 5, "type": "ray", "error": 0, "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "611d7d8b4b40eb8362cca23301000000", + "ray.actor_id": "690dcee46cf26e93ee89e59001000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", - "ray.task_id": "f91b78d7db9a6593611d7d8b4b40eb8362cca23301000000", - "ray.worker_id": "9a4dda773aba8c8fe80ba86b4ef71e66564f81bcb25edd2900f3a643", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.task_id": "85748392bcd969cc690dcee46cf26e93ee89e59001000000", + "ray.worker_id": "75bc236cdf71c7b5a1a4fa31b673d7f14572c4437fbc95fcedec2fa5", "span.kind": "consumer" }, "metrics": { @@ -191,12 +195,12 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 63291, - "start": 1764257369243536881 + "duration": 42750, + "start": 1764326814911251418 }, { "name": "exec entrypoint", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "exec task_without_init.py", "trace_id": 0, "span_id": 6, @@ -206,13 +210,13 @@ "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "611d7d8b4b40eb8362cca23301000000", + "ray.actor_id": "690dcee46cf26e93ee89e59001000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", - "ray.task_id": "f91b78d7db9a6593611d7d8b4b40eb8362cca23301000000", - "ray.worker_id": "9a4dda773aba8c8fe80ba86b4ef71e66564f81bcb25edd2900f3a643", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.task_id": "85748392bcd969cc690dcee46cf26e93ee89e59001000000", + "ray.worker_id": "75bc236cdf71c7b5a1a4fa31b673d7f14572c4437fbc95fcedec2fa5", "span.kind": "consumer" }, "metrics": { @@ -222,12 +226,12 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 482000, - "start": 1764257369243832256 + "duration": 401791, + "start": 1764326814911467793 }, { "name": "task.submit", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "__main__.add_one.remote", "trace_id": 0, "span_id": 7, @@ -236,20 +240,17 @@ "error": 0, "meta": { "_dd.base_service": "project", - "_dd.p.tid": "69286e5800000000", - "_dd.parent_id": "2cd921d88a156042", + "_dd.p.tid": "69297d9d00000000", + "_dd.parent_id": "c9a7f2428679c895", "component": "ray", "language": "python", "ray.hostname": "docker-desktop", - "ray.job_id": "03000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", "ray.task.submit_status": "success", - "ray.worker_id": "03000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "e6f91ed4f46641bb93e5625bb348d5d6", + "runtime-id": "3f5cf2f03db844538d5bdfb2f5aca662", "span.kind": "producer", - "traceparent": "00-69286e5800000000abceee630a6a000f-2cd921d88a156042-00", - "tracestate": "dd=p:2cd921d88a156042;s:0" + "traceparent": "00-69297d9d00000000bf349262b273a9f9-c9a7f2428679c895-00", + "tracestate": "dd=p:c9a7f2428679c895;s:0" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -259,15 +260,55 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1888.0 + "process_id": 2209.0 }, - "duration": 232583, - "start": 1764257369740446173 + "duration": 46862458, + "start": 1764326815281651293 }, + { + "name": "task.execute", + "service": "unnamed.ray.job", + "resource": "__main__.add_one", + "trace_id": 0, + "span_id": 12, + "parent_id": 7, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "_dd.p.tid": "69297d9d00000000", + "_dd.parent_id": "c9020337cd482b77", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "04000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.task.status": "success", + "ray.task_id": "ca80f3a8e8ba2e50ffffffffffffffffffffffff04000000", + "ray.worker_id": "92ca8487cabc419e929466bc527769c1d49f235d58a0cde361b95018", + "runtime-id": "51b38c234ba24001a5c0360c6ddf2a6e", + "span.kind": "consumer", + "traceparent": "00-69297d9d00000000bf349262b273a9f9-c9020337cd482b77-00", + "tracestate": "dd=p:c9020337cd482b77;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 2257.0 + }, + "duration": 233875, + "start": 1764326816172440293 + }, { "name": "task.submit", - "service": "ray.dashboard", - "resource": "__main__.add_two.remote", + "service": "unnamed.ray.job", + "resource": "__main__.add_one.remote", "trace_id": 0, "span_id": 8, "parent_id": 4, @@ -275,20 +316,20 @@ "error": 0, "meta": { "_dd.base_service": "project", - "_dd.p.tid": "69286e5800000000", - "_dd.parent_id": "2cd921d88a156042", + "_dd.p.tid": "69297d9d00000000", + "_dd.parent_id": "c9a7f2428679c895", "component": "ray", "language": "python", "ray.hostname": "docker-desktop", - "ray.job_id": "03000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.job_id": "04000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", "ray.task.submit_status": "success", - "ray.worker_id": "03000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "e6f91ed4f46641bb93e5625bb348d5d6", + "ray.worker_id": "04000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "3f5cf2f03db844538d5bdfb2f5aca662", "span.kind": "producer", - "traceparent": "00-69286e5800000000abceee630a6a000f-2cd921d88a156042-00", - "tracestate": "dd=p:2cd921d88a156042;s:0" + "traceparent": "00-69297d9d00000000bf349262b273a9f9-c9a7f2428679c895-00", + "tracestate": "dd=p:c9a7f2428679c895;s:0" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -298,37 +339,37 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1888.0 + "process_id": 2209.0 }, - "duration": 1459625, - "start": 1764257370473452465 + "duration": 192875, + "start": 1764326815329576293 }, { "name": "task.execute", - "service": "ray.dashboard", - "resource": "__main__.add_two", + "service": "unnamed.ray.job", + "resource": "__main__.add_one", "trace_id": 0, - "span_id": 11, + "span_id": 13, "parent_id": 8, "type": "ray", "error": 0, "meta": { "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69286e5800000000", - "_dd.parent_id": "8d237abcbfe86ed1", + "_dd.p.tid": "69297d9d00000000", + "_dd.parent_id": "09f655a6cbe5baf5", "component": "ray", "language": "python", "ray.hostname": "docker-desktop", - "ray.job_id": "03000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.job_id": "04000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", "ray.task.status": "success", - "ray.task_id": "0ab01f2d6283d719ffffffffffffffffffffffff03000000", - "ray.worker_id": "ee8139d01a3b0958ba02bebad045b206719d8bd9d3fc6193768e57e6", - "runtime-id": "4c250db564d64507a00fe48a38ce5996", + "ray.task_id": "d8c1ad1586102897ffffffffffffffffffffffff04000000", + "ray.worker_id": "92ca8487cabc419e929466bc527769c1d49f235d58a0cde361b95018", + "runtime-id": "51b38c234ba24001a5c0360c6ddf2a6e", "span.kind": "consumer", - "traceparent": "00-69286e5800000000abceee630a6a000f-8d237abcbfe86ed1-00", - "tracestate": "dd=p:8d237abcbfe86ed1;s:0" + "traceparent": "00-69297d9d00000000bf349262b273a9f9-09f655a6cbe5baf5-00", + "tracestate": "dd=p:09f655a6cbe5baf5;s:0" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -338,14 +379,14 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1936.0 + "process_id": 2257.0 }, - "duration": 273667, - "start": 1764257370476360006 + "duration": 127666, + "start": 1764326816174502877 }, { "name": "task.submit", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "__main__.add_two.remote", "trace_id": 0, "span_id": 9, @@ -354,20 +395,20 @@ "error": 0, "meta": { "_dd.base_service": "project", - "_dd.p.tid": "69286e5800000000", - "_dd.parent_id": "2cd921d88a156042", + "_dd.p.tid": "69297d9d00000000", + "_dd.parent_id": "c9a7f2428679c895", "component": "ray", "language": "python", "ray.hostname": "docker-desktop", - "ray.job_id": "03000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.job_id": "04000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", "ray.task.submit_status": "success", - "ray.worker_id": "03000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "e6f91ed4f46641bb93e5625bb348d5d6", + "ray.worker_id": "04000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "3f5cf2f03db844538d5bdfb2f5aca662", "span.kind": "producer", - "traceparent": "00-69286e5800000000abceee630a6a000f-2cd921d88a156042-00", - "tracestate": "dd=p:2cd921d88a156042;s:0" + "traceparent": "00-69297d9d00000000bf349262b273a9f9-c9a7f2428679c895-00", + "tracestate": "dd=p:c9a7f2428679c895;s:0" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -377,37 +418,116 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1888.0 + "process_id": 2209.0 }, - "duration": 225959, - "start": 1764257370475828131 + "duration": 1140167, + "start": 1764326816175653460 }, { "name": "task.execute", - "service": "ray.dashboard", + "service": "unnamed.ray.job", "resource": "__main__.add_two", "trace_id": 0, - "span_id": 12, + "span_id": 14, "parent_id": 9, "type": "ray", "error": 0, "meta": { "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69286e5800000000", - "_dd.parent_id": "25ff86ae80db82af", + "_dd.p.tid": "69297d9d00000000", + "_dd.parent_id": "9a61c082c74ff91f", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "04000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.task.status": "success", + "ray.task_id": "5da13aa9e2150759ffffffffffffffffffffffff04000000", + "ray.worker_id": "92ca8487cabc419e929466bc527769c1d49f235d58a0cde361b95018", + "runtime-id": "51b38c234ba24001a5c0360c6ddf2a6e", + "span.kind": "consumer", + "traceparent": "00-69297d9d00000000bf349262b273a9f9-9a61c082c74ff91f-00", + "tracestate": "dd=p:9a61c082c74ff91f;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 2257.0 + }, + "duration": 148125, + "start": 1764326816178042418 + }, + { + "name": "task.submit", + "service": "unnamed.ray.job", + "resource": "__main__.add_two.remote", + "trace_id": 0, + "span_id": 10, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "project", + "_dd.p.tid": "69297d9d00000000", + "_dd.parent_id": "c9a7f2428679c895", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "04000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.task.submit_status": "success", + "ray.worker_id": "04000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "3f5cf2f03db844538d5bdfb2f5aca662", + "span.kind": "producer", + "traceparent": "00-69297d9d00000000bf349262b273a9f9-c9a7f2428679c895-00", + "tracestate": "dd=p:c9a7f2428679c895;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 2209.0 + }, + "duration": 150042, + "start": 1764326816177453460 + }, + { + "name": "task.execute", + "service": "unnamed.ray.job", + "resource": "__main__.add_two", + "trace_id": 0, + "span_id": 15, + "parent_id": 10, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "ray._private.workers", + "_dd.p.tid": "69297d9d00000000", + "_dd.parent_id": "b9042c2e4e83c5b7", "component": "ray", "language": "python", "ray.hostname": "docker-desktop", - "ray.job_id": "03000000", - "ray.node_id": "4e88799d50e054183c7ac90fb4353445a408584cd2f3849934de4518", - "ray.submission_id": "raysubmit_upnmSxwiQ9DvDKfT", + "ray.job_id": "04000000", + "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", "ray.task.status": "success", - "ray.task_id": "62ffec03f52574bdffffffffffffffffffffffff03000000", - "ray.worker_id": "ee8139d01a3b0958ba02bebad045b206719d8bd9d3fc6193768e57e6", - "runtime-id": "4c250db564d64507a00fe48a38ce5996", + "ray.task_id": "011ae9488efe4e25ffffffffffffffffffffffff04000000", + "ray.worker_id": "92ca8487cabc419e929466bc527769c1d49f235d58a0cde361b95018", + "runtime-id": "51b38c234ba24001a5c0360c6ddf2a6e", "span.kind": "consumer", - "traceparent": "00-69286e5800000000abceee630a6a000f-25ff86ae80db82af-00", - "tracestate": "dd=p:25ff86ae80db82af;s:0" + "traceparent": "00-69297d9d00000000bf349262b273a9f9-b9042c2e4e83c5b7-00", + "tracestate": "dd=p:b9042c2e4e83c5b7;s:0" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -417,8 +537,8 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1936.0 + "process_id": 2257.0 }, - "duration": 138750, - "start": 1764257370478251923 + "duration": 106583, + "start": 1764326816179173585 }]] diff --git a/tests/snapshots/tests.contrib.ray.test_ray.test_service_name.json b/tests/snapshots/tests.contrib.ray.test_ray.test_service_name.json new file mode 100644 index 00000000000..58698de0a2b --- /dev/null +++ b/tests/snapshots/tests.contrib.ray.test_ray.test_service_name.json @@ -0,0 +1,298 @@ +[[ + { + "name": "ray.job", + "service": "test", + "resource": "ray.job", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "ray", + "error": 0, + "meta": { + "_dd.p.dm": "-0", + "_dd.p.tid": "69297dc100000000", + "component": "ray", + "language": "python", + "ray.entrypoint": "python /service.py", + "ray.hostname": "docker-desktop", + "ray.job.message": "Job finished successfully.", + "ray.job.metadata.foo": "bar", + "ray.job.status": "SUCCEEDED", + "ray.job_id": "01000000", + "ray.node_id": "28449ca12072a95949a446c16546f92f183a7fa20b4e9ecfe80a3021", + "ray.submission_id": "raysubmit_EceQv4k8jb6z19RK", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "6f2e0111f5f04f928ee1b56f68b1afee" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 2890.0 + }, + "duration": 4005359335, + "start": 1764326849603570045 + }, + { + "name": "ray.job.submit", + "service": "test", + "resource": "ray.job.submit", + "trace_id": 0, + "span_id": 2, + "parent_id": 1, + "type": "ray", + "error": 0, + "meta": { + "component": "ray", + "ray.hostname": "docker-desktop", + "ray.job.submit_status": "success", + "ray.job_id": "01000000", + "ray.node_id": "28449ca12072a95949a446c16546f92f183a7fa20b4e9ecfe80a3021", + "ray.submission_id": "raysubmit_EceQv4k8jb6z19RK", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "span.kind": "producer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 68166, + "start": 1764326849603756462 + }, + { + "name": "actor_method.submit", + "service": "test", + "resource": "JobSupervisor.run.remote", + "trace_id": 0, + "span_id": 3, + "parent_id": 1, + "type": "ray", + "error": 0, + "meta": { + "component": "ray", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "28449ca12072a95949a446c16546f92f183a7fa20b4e9ecfe80a3021", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "span.kind": "producer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 218084, + "start": 1764326849646427628 + }, + { + "name": "actor_method.execute", + "service": "test", + "resource": "JobSupervisor.run", + "trace_id": 0, + "span_id": 4, + "parent_id": 3, + "type": "ray", + "error": 0, + "meta": { + "_dd.p.tid": "69297dc100000000", + "_dd.parent_id": "6be983922fdb12ce", + "component": "ray", + "language": "python", + "ray.actor_id": "31afeef11bce8268583adf9b01000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "28449ca12072a95949a446c16546f92f183a7fa20b4e9ecfe80a3021", + "ray.submission_id": "raysubmit_EceQv4k8jb6z19RK", + "ray.task_id": "16310a0f0a45af5c31afeef11bce8268583adf9b01000000", + "ray.worker_id": "c018ec9d3f07455470fd587cedcfc6c1649227c3a58d3d9f685304cc", + "runtime-id": "886f8cb35adb467a8c604e636b35a8b5", + "span.kind": "consumer", + "traceparent": "00-69297dc100000000d0c3b29f1585ea8f-6be983922fdb12ce-00", + "tracestate": "dd=p:6be983922fdb12ce" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 3076.0 + }, + "duration": 2045583834, + "start": 1764326850579168504 + }, + { + "name": "actor_method.execute", + "service": "test", + "resource": "JobSupervisor._get_driver_env_vars", + "trace_id": 0, + "span_id": 5, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "component": "ray", + "ray.actor_id": "31afeef11bce8268583adf9b01000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "28449ca12072a95949a446c16546f92f183a7fa20b4e9ecfe80a3021", + "ray.submission_id": "raysubmit_EceQv4k8jb6z19RK", + "ray.task_id": "16310a0f0a45af5c31afeef11bce8268583adf9b01000000", + "ray.worker_id": "c018ec9d3f07455470fd587cedcfc6c1649227c3a58d3d9f685304cc", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 1840292, + "start": 1764326850580341420 + }, + { + "name": "actor_method.execute", + "service": "test", + "resource": "JobSupervisor._get_driver_runtime_env", + "trace_id": 0, + "span_id": 8, + "parent_id": 5, + "type": "ray", + "error": 0, + "meta": { + "component": "ray", + "ray.actor_id": "31afeef11bce8268583adf9b01000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "28449ca12072a95949a446c16546f92f183a7fa20b4e9ecfe80a3021", + "ray.submission_id": "raysubmit_EceQv4k8jb6z19RK", + "ray.task_id": "16310a0f0a45af5c31afeef11bce8268583adf9b01000000", + "ray.worker_id": "c018ec9d3f07455470fd587cedcfc6c1649227c3a58d3d9f685304cc", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 45666, + "start": 1764326850582113004 + }, + { + "name": "exec entrypoint", + "service": "test", + "resource": "exec service.py", + "trace_id": 0, + "span_id": 6, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "component": "ray", + "ray.actor_id": "31afeef11bce8268583adf9b01000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "28449ca12072a95949a446c16546f92f183a7fa20b4e9ecfe80a3021", + "ray.submission_id": "raysubmit_EceQv4k8jb6z19RK", + "ray.task_id": "16310a0f0a45af5c31afeef11bce8268583adf9b01000000", + "ray.worker_id": "c018ec9d3f07455470fd587cedcfc6c1649227c3a58d3d9f685304cc", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 342750, + "start": 1764326850582349170 + }, + { + "name": "task.submit", + "service": "test", + "resource": "__main__.add_one.remote", + "trace_id": 0, + "span_id": 7, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.p.tid": "69297dc100000000", + "_dd.parent_id": "5fb293f886f48728", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.submission_id": "raysubmit_EceQv4k8jb6z19RK", + "ray.task.submit_status": "success", + "runtime-id": "3bc641965f264ae5a7bb6032ffdf1c41", + "span.kind": "producer", + "traceparent": "00-69297dc100000000d0c3b29f1585ea8f-5fb293f886f48728-00", + "tracestate": "dd=p:5fb293f886f48728;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 3133.0 + }, + "duration": 50276666, + "start": 1764326850951673546 + }, + { + "name": "task.execute", + "service": "test", + "resource": "__main__.add_one", + "trace_id": 0, + "span_id": 9, + "parent_id": 7, + "type": "ray", + "error": 0, + "meta": { + "_dd.p.tid": "69297dc100000000", + "_dd.parent_id": "e57fd4b767e07154", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "02000000", + "ray.node_id": "28449ca12072a95949a446c16546f92f183a7fa20b4e9ecfe80a3021", + "ray.submission_id": "raysubmit_EceQv4k8jb6z19RK", + "ray.task.status": "success", + "ray.task_id": "67a2e8cfa5a06db3ffffffffffffffffffffffff02000000", + "ray.worker_id": "ded1e87b68ceed271a703bdf5b2c5548cdacfc2d16a1ea27b19ea36e", + "runtime-id": "3515bd84dfda43a68748167e3bb226e0", + "span.kind": "consumer", + "traceparent": "00-69297dc100000000d0c3b29f1585ea8f-e57fd4b767e07154-00", + "tracestate": "dd=p:e57fd4b767e07154;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 3181.0 + }, + "duration": 217416, + "start": 1764326851813965213 + }]] diff --git a/tests/snapshots/tests.contrib.ray.test_ray.test_task_without_init.json b/tests/snapshots/tests.contrib.ray.test_ray.test_task_without_init.json deleted file mode 100644 index fc56dd05771..00000000000 --- a/tests/snapshots/tests.contrib.ray.test_ray.test_task_without_init.json +++ /dev/null @@ -1,69 +0,0 @@ -[[ - { - "name": "task.submit", - "service": "tests.contrib.ray", - "resource": "tests.contrib.ray.test_ray.add_one.remote", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "ray", - "error": 0, - "meta": { - "_dd.p.dm": "-0", - "_dd.p.tid": "69025c6200000000", - "component": "ray", - "language": "python", - "ray.hostname": "docker-desktop", - "ray.task.submit_status": "success", - "runtime-id": "467a8549e29149d3a0a561b9407dc4f6", - "span.kind": "producer" - }, - "metrics": { - "_dd.ai_obs.enabled": 1, - "_dd.djm.enabled": 1, - "_dd.filter.kept": 1, - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 2, - "process_id": 605 - }, - "duration": 3830005877, - "start": 1761762402453522879 - }], -[ - { - "name": "task.submit", - "service": "tests.contrib.ray", - "resource": "tests.contrib.ray.test_ray.add_one.remote", - "trace_id": 1, - "span_id": 1, - "parent_id": 0, - "type": "ray", - "error": 0, - "meta": { - "_dd.p.dm": "-0", - "_dd.p.tid": "69025c6600000000", - "component": "ray", - "language": "python", - "ray.hostname": "docker-desktop", - "ray.job_id": "01000000", - "ray.node_id": "9221982158053df5fa3e8b7cfe1f487b8aef5fb4b18f9fcfc3a76944", - "ray.task.submit_status": "success", - "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "467a8549e29149d3a0a561b9407dc4f6", - "span.kind": "producer" - }, - "metrics": { - "_dd.ai_obs.enabled": 1, - "_dd.djm.enabled": 1, - "_dd.filter.kept": 1, - "_dd.measured": 1, - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 2, - "process_id": 605 - }, - "duration": 389792, - "start": 1761762406283924423 - }]] diff --git a/tests/snapshots/tests.contrib.ray.test_ray.test_use_entrypoint_service_name.json b/tests/snapshots/tests.contrib.ray.test_ray.test_use_entrypoint_service_name.json new file mode 100644 index 00000000000..7e381dc2a5c --- /dev/null +++ b/tests/snapshots/tests.contrib.ray.test_ray.test_use_entrypoint_service_name.json @@ -0,0 +1,307 @@ +[[ + { + "name": "ray.job", + "service": "service", + "resource": "ray.job", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "test", + "_dd.p.dm": "-0", + "_dd.p.tid": "69297de500000000", + "component": "ray", + "language": "python", + "ray.entrypoint": "python /service.py", + "ray.hostname": "docker-desktop", + "ray.job.message": "Job finished successfully.", + "ray.job.metadata.foo": "bar", + "ray.job.status": "SUCCEEDED", + "ray.job_id": "01000000", + "ray.node_id": "8f82f83e5e0c40e7e9e15ef6fdc81f8d9f5c690f3a387ea85bb87dd6", + "ray.submission_id": "raysubmit_TJUQtnU1K7MpVuft", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "runtime-id": "ca8fb23a01db4f8faf642770612080c9" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 3837.0 + }, + "duration": 3961365419, + "start": 1764326885341548964 + }, + { + "name": "ray.job.submit", + "service": "service", + "resource": "ray.job.submit", + "trace_id": 0, + "span_id": 2, + "parent_id": 1, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "test", + "component": "ray", + "ray.hostname": "docker-desktop", + "ray.job.submit_status": "success", + "ray.job_id": "01000000", + "ray.node_id": "8f82f83e5e0c40e7e9e15ef6fdc81f8d9f5c690f3a387ea85bb87dd6", + "ray.submission_id": "raysubmit_TJUQtnU1K7MpVuft", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "span.kind": "producer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 64375, + "start": 1764326885341738339 + }, + { + "name": "actor_method.submit", + "service": "service", + "resource": "JobSupervisor.run.remote", + "trace_id": 0, + "span_id": 3, + "parent_id": 1, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "test", + "component": "ray", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "8f82f83e5e0c40e7e9e15ef6fdc81f8d9f5c690f3a387ea85bb87dd6", + "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", + "span.kind": "producer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 238500, + "start": 1764326885387938298 + }, + { + "name": "actor_method.execute", + "service": "service", + "resource": "JobSupervisor.run", + "trace_id": 0, + "span_id": 4, + "parent_id": 3, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "test", + "_dd.p.tid": "69297de500000000", + "_dd.parent_id": "11bdb3a8740df751", + "component": "ray", + "language": "python", + "ray.actor_id": "b085001c8e35bfb141d2a76101000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "8f82f83e5e0c40e7e9e15ef6fdc81f8d9f5c690f3a387ea85bb87dd6", + "ray.submission_id": "raysubmit_TJUQtnU1K7MpVuft", + "ray.task_id": "16310a0f0a45af5cb085001c8e35bfb141d2a76101000000", + "ray.worker_id": "1feb288a7f84a2138cfe97af32c8fe816ae999f71a9aa5d871a93f78", + "runtime-id": "2cf89cb6fcf94645a7532eeaa511f27a", + "span.kind": "consumer", + "traceparent": "00-69297de500000000580cb316d0c3e798-11bdb3a8740df751-00", + "tracestate": "dd=p:11bdb3a8740df751" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 4023.0 + }, + "duration": 2031432751, + "start": 1764326886279836381 + }, + { + "name": "actor_method.execute", + "service": "service", + "resource": "JobSupervisor._get_driver_env_vars", + "trace_id": 0, + "span_id": 5, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "test", + "component": "ray", + "ray.actor_id": "b085001c8e35bfb141d2a76101000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "8f82f83e5e0c40e7e9e15ef6fdc81f8d9f5c690f3a387ea85bb87dd6", + "ray.submission_id": "raysubmit_TJUQtnU1K7MpVuft", + "ray.task_id": "16310a0f0a45af5cb085001c8e35bfb141d2a76101000000", + "ray.worker_id": "1feb288a7f84a2138cfe97af32c8fe816ae999f71a9aa5d871a93f78", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 2191291, + "start": 1764326886281202840 + }, + { + "name": "actor_method.execute", + "service": "service", + "resource": "JobSupervisor._get_driver_runtime_env", + "trace_id": 0, + "span_id": 8, + "parent_id": 5, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "test", + "component": "ray", + "ray.actor_id": "b085001c8e35bfb141d2a76101000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "8f82f83e5e0c40e7e9e15ef6fdc81f8d9f5c690f3a387ea85bb87dd6", + "ray.submission_id": "raysubmit_TJUQtnU1K7MpVuft", + "ray.task_id": "16310a0f0a45af5cb085001c8e35bfb141d2a76101000000", + "ray.worker_id": "1feb288a7f84a2138cfe97af32c8fe816ae999f71a9aa5d871a93f78", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 41833, + "start": 1764326886283331090 + }, + { + "name": "exec entrypoint", + "service": "service", + "resource": "exec service.py", + "trace_id": 0, + "span_id": 6, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "test", + "component": "ray", + "ray.actor_id": "b085001c8e35bfb141d2a76101000000", + "ray.hostname": "docker-desktop", + "ray.job_id": "01000000", + "ray.node_id": "8f82f83e5e0c40e7e9e15ef6fdc81f8d9f5c690f3a387ea85bb87dd6", + "ray.submission_id": "raysubmit_TJUQtnU1K7MpVuft", + "ray.task_id": "16310a0f0a45af5cb085001c8e35bfb141d2a76101000000", + "ray.worker_id": "1feb288a7f84a2138cfe97af32c8fe816ae999f71a9aa5d871a93f78", + "span.kind": "consumer" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_sampling_priority_v1": 2.0 + }, + "duration": 405500, + "start": 1764326886283533006 + }, + { + "name": "task.submit", + "service": "service", + "resource": "__main__.add_one.remote", + "trace_id": 0, + "span_id": 7, + "parent_id": 4, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "test", + "_dd.p.tid": "69297de500000000", + "_dd.parent_id": "54668968e28dc86e", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.submission_id": "raysubmit_TJUQtnU1K7MpVuft", + "ray.task.submit_status": "success", + "runtime-id": "714c620f1a0842ffa616d58cafacdc68", + "span.kind": "producer", + "traceparent": "00-69297de500000000580cb316d0c3e798-54668968e28dc86e-00", + "tracestate": "dd=p:54668968e28dc86e;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 4080.0 + }, + "duration": 49940750, + "start": 1764326886654194298 + }, + { + "name": "task.execute", + "service": "service", + "resource": "__main__.add_one", + "trace_id": 0, + "span_id": 9, + "parent_id": 7, + "type": "ray", + "error": 0, + "meta": { + "_dd.base_service": "test", + "_dd.p.tid": "69297de500000000", + "_dd.parent_id": "36319e584a84c9b7", + "component": "ray", + "language": "python", + "ray.hostname": "docker-desktop", + "ray.job_id": "02000000", + "ray.node_id": "8f82f83e5e0c40e7e9e15ef6fdc81f8d9f5c690f3a387ea85bb87dd6", + "ray.submission_id": "raysubmit_TJUQtnU1K7MpVuft", + "ray.task.status": "success", + "ray.task_id": "67a2e8cfa5a06db3ffffffffffffffffffffffff02000000", + "ray.worker_id": "e74f9df9f692ae2bf646e804b5a43bc917207e8f6120b2d7a0af4b95", + "runtime-id": "7dd6bfb16057402ab0d4ccea58839d55", + "span.kind": "consumer", + "traceparent": "00-69297de500000000580cb316d0c3e798-36319e584a84c9b7-00", + "tracestate": "dd=p:36319e584a84c9b7;s:0" + }, + "metrics": { + "_dd.ai_obs.enabled": 1.0, + "_dd.djm.enabled": 1.0, + "_dd.filter.kept": 1.0, + "_dd.measured": 1.0, + "_dd.top_level": 1.0, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 2.0, + "process_id": 4128.0 + }, + "duration": 248833, + "start": 1764326887545712507 + }]] From 96d9186f8b1a4e6e1214b05a29fe6bd7809223db Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Fri, 28 Nov 2025 13:34:23 +0100 Subject: [PATCH 4/8] add denylist for task submission --- ddtrace/contrib/internal/ray/patch.py | 15 ++++++++++++--- tests/contrib/ray/test_ray.py | 17 ++++++++++++++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/ddtrace/contrib/internal/ray/patch.py b/ddtrace/contrib/internal/ray/patch.py index f0196e22e60..f25b4952788 100644 --- a/ddtrace/contrib/internal/ray/patch.py +++ b/ddtrace/contrib/internal/ray/patch.py @@ -68,8 +68,14 @@ RAY_SERVICE_NAME = os.environ.get(RAY_JOB_NAME) # Ray modules that should be excluded from tracing -RAY_MODULE_DENYLIST = { - "ray.dag", +RAY_COMMON_MODULE_DENYLIST = { + "ray.data._internal", +} + +RAY_TASK_MODULE_DENYLIST = {*RAY_COMMON_MODULE_DENYLIST} + +RAY_ACTOR_MODULE_DENYLIST = { + *RAY_COMMON_MODULE_DENYLIST, "ray.experimental", "ray.data._internal", } @@ -143,6 +149,9 @@ def _wrap_task_execution(wrapped, *args, **kwargs): def traced_submit_task(wrapped, instance, args, kwargs): """Trace task submission, i.e the func.remote() call""" + if instance._function.__module__ in RAY_TASK_MODULE_DENYLIST: + return wrapped(*args, **kwargs) + if tracer.current_span() is None: log.debug( "No active span found in %s.remote(), activating trace context from environment", instance._function_name @@ -498,7 +507,7 @@ def inject_tracing_into_actor_class(wrapped, instance, args, kwargs): class_name = str(cls.__name__) # Skip tracing for certain ray modules - if any(module_name.startswith(denied_module) for denied_module in RAY_MODULE_DENYLIST): + if any(module_name.startswith(denied_module) for denied_module in RAY_ACTOR_MODULE_DENYLIST): return cls # Actor beginning with _ are considered internal and will not be traced diff --git a/tests/contrib/ray/test_ray.py b/tests/contrib/ray/test_ray.py index c3834b0f6c9..ace2ffde895 100644 --- a/tests/contrib/ray/test_ray.py +++ b/tests/contrib/ray/test_ray.py @@ -106,7 +106,7 @@ def increment_get_and_double(self): assert current_value == 2, f"Unexpected result: {current_value}" - @pytest.mark.snapshot(token="tests.contrib.ray.test_ray.test_ignored_actored", ignores=RAY_SNAPSHOT_IGNORES) + @pytest.mark.snapshot(token="tests.contrib.ray.test_ray.test_ignored_actors", ignores=RAY_SNAPSHOT_IGNORES) def test_ignored_actors(self): @ray.remote class _InternalActor: @@ -128,6 +128,21 @@ def get_value(self): value = ray.get(denied_actor.get_value.remote()) assert value == 42, f"Unexpected result: {value}" + @pytest.mark.snapshot(token="tests.contrib.ray.test_ray.test_ignored_tasks", ignores=RAY_SNAPSHOT_IGNORES) + def test_ignored_tasks(self): + """Test that tasks from modules in RAY_TASK_MODULE_DENYLIST are not traced.""" + + def mock_denied_task(x): + return x + 1 + + # Set the module to be in the denylist + mock_denied_task.__module__ = "ray.data._internal" + mock_denied_task = ray.remote(mock_denied_task) + + # Submit the task and verify it executes correctly + result = ray.get(mock_denied_task.remote(41)) + assert result == 42, f"Unexpected result: {result}" + @pytest.mark.snapshot(token="tests.contrib.ray.test_ray.test_nested_tasks", ignores=RAY_SNAPSHOT_IGNORES) def test_nested_tasks(self): @ray.remote From 50301c6d39382a0f4517598f51ac131b9904078e Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Fri, 28 Nov 2025 16:57:59 +0100 Subject: [PATCH 5/8] fix a potential issue --- ddtrace/contrib/internal/ray/patch.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ddtrace/contrib/internal/ray/patch.py b/ddtrace/contrib/internal/ray/patch.py index f25b4952788..3e3a3b24be4 100644 --- a/ddtrace/contrib/internal/ray/patch.py +++ b/ddtrace/contrib/internal/ray/patch.py @@ -390,12 +390,12 @@ def traced_wait(wrapped, instance, args, kwargs): def _job_supervisor_run_wrapper(method: Callable[..., Any]) -> Any: - async def _traced_run_method(self: Any, *args: Any, _dd_ray_trace_ctx, **kwargs: Any) -> Any: + async def _traced_run_method(self: Any, *args: Any, _dd_ray_trace_ctx=None, **kwargs: Any) -> Any: import ray.exceptions from ddtrace.ext import SpanTypes - context = _TraceContext._extract(_dd_ray_trace_ctx) + context = _TraceContext._extract(_dd_ray_trace_ctx) if _dd_ray_trace_ctx else None submission_id = os.environ.get(RAY_SUBMISSION_ID) with long_running_ray_span( @@ -567,6 +567,10 @@ def patch(): ray._datadog_patch = True + from ray.util.tracing import tracing_helper + + tracing_helper._global_is_tracing_enabled = False + @ModuleWatchdog.after_module_imported("ray.actor") def _(m): _w(m.ActorHandle, "_actor_method_call", traced_actor_method_call) From 3c8f4c8de9cbc1bb378d41d817cb48720af6006c Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Mon, 1 Dec 2025 14:00:46 +0100 Subject: [PATCH 6/8] fix snapshots --- ddtrace/contrib/internal/ray/__init__.py | 8 +- tests/contrib/ray/test_ray.py | 42 ++- ...ayWithoutInit.test_actor_without_init.json | 178 +++------- ...ayWithoutInit.test_job_name_specified.json | 176 +++------- ...RayWithoutInit.test_task_without_init.json | 330 ++++++------------ 5 files changed, 243 insertions(+), 491 deletions(-) diff --git a/ddtrace/contrib/internal/ray/__init__.py b/ddtrace/contrib/internal/ray/__init__.py index 33e6ce1ca00..eb32b318329 100644 --- a/ddtrace/contrib/internal/ray/__init__.py +++ b/ddtrace/contrib/internal/ray/__init__.py @@ -10,15 +10,13 @@ Ray instrumentation is experimental. It is deactivated by default. To enable it, you have to follow one of the two methods below: -The recommended way to instrument Ray, is to instrument the Ray cluster. - -You can do it by starting the Ray head using ddtrace-run:: +The recommended way to instrument Ray, is to instrument the Ray cluster using ddtrace-run:: DD_PATCH_MODULES="ray:true, aiohttp:false, grpc:false, requests:false" ddtrace-run ray start --head -DD_PATCH_MODULES will allow to reduce noise. +DD_PATCH_MODULES will allow to reduce noise by sending only the jobs related spans. -You can do it by starting the Ray head with a tracing startup hook:: +You can also do it by starting Ray head with a tracing startup hook:: ray start --head --tracing-startup-hook=ddtrace.contrib.ray:setup_tracing diff --git a/tests/contrib/ray/test_ray.py b/tests/contrib/ray/test_ray.py index ace2ffde895..3577d497157 100644 --- a/tests/contrib/ray/test_ray.py +++ b/tests/contrib/ray/test_ray.py @@ -330,7 +330,7 @@ def increment(self, x, y=1): assert current_value == 3, f"Unexpected result: {current_value}" -def _start_ray_cluster(env_vars=None): +def _start_ray_cluster(env_vars=None, ddtrace_run=False): """Start a Ray cluster with optional environment variables.""" env = os.environ.copy() base_env = { @@ -340,9 +340,12 @@ def _start_ray_cluster(env_vars=None): base_env.update(env_vars) env.update(base_env) - subprocess.run( + cmd = [] + if ddtrace_run: + cmd.append("ddtrace-run") + + cmd.extend( [ - "ddtrace-run", "ray", "start", "--head", @@ -351,7 +354,13 @@ def _start_ray_cluster(env_vars=None): "--object-store-memory=100000000", "--dashboard-host=127.0.0.1", "--dashboard-port=8265", - ], + ] + ) + if not ddtrace_run: + cmd.append("--tracing-startup-hook=ddtrace.contrib.ray:setup_tracing") + + subprocess.run( + cmd, env=env, check=True, ) @@ -399,9 +408,11 @@ def _submit_and_wait_for_job(dashboard_url, job_script_name, metadata={"foo": "b class TestRayWithoutInit(TracerTestCase): - """This class tests that actor and task submissions work with auto initialization - This class also shows a lack of visibility when we cannot call ray.init explictly - which led to a change in how we start a cluster + """This test Class is used for two purposes: + - Show a lack of observability when relying on automatic ray.init() + see test_task_without_init and test_actor_without_init(). + - Give examples a real traces (because we create a real cluster) + - Show some behavior of service naming: default one, using metadata """ dashboard_url = None @@ -418,21 +429,33 @@ def tearDownClass(cls): @pytest.mark.snapshot(ignores=RAY_SNAPSHOT_IGNORES) def test_task_without_init(self): + """The first submission triggers ray.init() so it is not traced + Therefore the execution cannot be instrumented and we will see only + the second submission in the snapshot + + The second remote function is fully executed + """ + _submit_and_wait_for_job(self.dashboard_url, "task_without_init.py") @pytest.mark.snapshot(ignores=RAY_SNAPSHOT_IGNORES) def test_actor_without_init(self): + """The creation of the actor triggers ray.init() so it cannot + be instrumented + """ + _submit_and_wait_for_job(self.dashboard_url, "actor_without_init.py") @pytest.mark.snapshot(ignores=RAY_SNAPSHOT_IGNORES) def test_job_name_specified(self): + """Check that the service name is set when specifying job name in the metadata""" _submit_and_wait_for_job(self.dashboard_url, "service.py", metadata={"job_name": "my_model"}) @pytest.mark.snapshot(ignores=RAY_SNAPSHOT_IGNORES) def test_service_name(): """Test that DD_SERVICE environment variable is used as service name.""" - dashboard_url = _start_ray_cluster(env_vars={"DD_SERVICE": "test"}) + dashboard_url = _start_ray_cluster(env_vars={"DD_SERVICE": "test"}, ddtrace_run=True) try: _submit_and_wait_for_job(dashboard_url, "service.py") @@ -447,7 +470,8 @@ def test_use_entrypoint_service_name(): env_vars={ "DD_SERVICE": "test", "DD_TRACE_RAY_USE_ENTRYPOINT_AS_SERVICE_NAME": "True", - } + }, + ddtrace_run=True, ) try: diff --git a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json index 8fb94faf027..6c2fc876552 100644 --- a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json +++ b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_actor_without_init.json @@ -11,7 +11,7 @@ "meta": { "_dd.base_service": "ray.dashboard", "_dd.p.dm": "-0", - "_dd.p.tid": "69297d9000000000", + "_dd.p.tid": "692da0cf00000000", "component": "ray", "language": "python", "ray.entrypoint": "python /actor_without_init.py", @@ -20,10 +20,10 @@ "ray.job.metadata.foo": "bar", "ray.job.status": "SUCCEEDED", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_MZJEkR9JaEyLTxRf", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "81f567481c4e41dc8f1b56ecf05d2973" + "runtime-id": "5f45deb5246d4c468a5bc92fb40b3998" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -33,10 +33,10 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1599.0 + "process_id": 1092.0 }, - "duration": 4046849919, - "start": 1764326800832845842 + "duration": 2850422918, + "start": 1764597967518769043 }, { "name": "ray.job.submit", @@ -53,8 +53,8 @@ "ray.hostname": "docker-desktop", "ray.job.submit_status": "success", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_MZJEkR9JaEyLTxRf", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", "span.kind": "producer" }, @@ -65,8 +65,8 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 59333, - "start": 1764326800833023842 + "duration": 61000, + "start": 1764597967518967668 }, { "name": "actor_method.submit", @@ -82,7 +82,7 @@ "component": "ray", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", "span.kind": "producer" }, @@ -93,8 +93,8 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 264542, - "start": 1764326800873390467 + "duration": 443834, + "start": 1764597967555345293 }, { "name": "actor_method.execute", @@ -107,21 +107,21 @@ "error": 0, "meta": { "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69297d9000000000", - "_dd.parent_id": "b8e28cc5ec16e6c9", + "_dd.p.tid": "692da0cf00000000", + "_dd.parent_id": "edef701ed62f584f", "component": "ray", "language": "python", - "ray.actor_id": "5d7b8f890a255711f2ceb2b401000000", + "ray.actor_id": "a703debb0a7c88156ee0838601000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", - "ray.task_id": "16310a0f0a45af5c5d7b8f890a255711f2ceb2b401000000", - "ray.worker_id": "08b10fdcbdb3706b74070824f41e97ce7ce7fb7f98b5ab24b6c69951", - "runtime-id": "9bd2dd0007f84438b2e83a857940d05d", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_MZJEkR9JaEyLTxRf", + "ray.task_id": "16310a0f0a45af5ca703debb0a7c88156ee0838601000000", + "ray.worker_id": "aa57a10c2ebca86dc84acdc592b944b64b2a68489a9c2b64c2f9de2d", + "runtime-id": "f5ec7b58657e4e9fb3f28411f4712dc6", "span.kind": "consumer", - "traceparent": "00-69297d90000000007b46d2532fc2b819-b8e28cc5ec16e6c9-00", - "tracestate": "dd=p:b8e28cc5ec16e6c9" + "traceparent": "00-692da0cf000000008f965ebbf3fb5229-edef701ed62f584f-00", + "tracestate": "dd=p:edef701ed62f584f" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -131,10 +131,10 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1787.0 + "process_id": 1269.0 }, - "duration": 2031085667, - "start": 1764326801860054676 + "duration": 1926620209, + "start": 1764597968360276502 }, { "name": "actor_method.execute", @@ -148,13 +148,13 @@ "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "5d7b8f890a255711f2ceb2b401000000", + "ray.actor_id": "a703debb0a7c88156ee0838601000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", - "ray.task_id": "16310a0f0a45af5c5d7b8f890a255711f2ceb2b401000000", - "ray.worker_id": "08b10fdcbdb3706b74070824f41e97ce7ce7fb7f98b5ab24b6c69951", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_MZJEkR9JaEyLTxRf", + "ray.task_id": "16310a0f0a45af5ca703debb0a7c88156ee0838601000000", + "ray.worker_id": "aa57a10c2ebca86dc84acdc592b944b64b2a68489a9c2b64c2f9de2d", "span.kind": "consumer" }, "metrics": { @@ -164,28 +164,28 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 1300375, - "start": 1764326801861597967 + "duration": 1184625, + "start": 1764597968361768169 }, { "name": "actor_method.execute", "service": "unnamed.ray.job", "resource": "JobSupervisor._get_driver_runtime_env", "trace_id": 0, - "span_id": 8, + "span_id": 7, "parent_id": 5, "type": "ray", "error": 0, "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "5d7b8f890a255711f2ceb2b401000000", + "ray.actor_id": "a703debb0a7c88156ee0838601000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", - "ray.task_id": "16310a0f0a45af5c5d7b8f890a255711f2ceb2b401000000", - "ray.worker_id": "08b10fdcbdb3706b74070824f41e97ce7ce7fb7f98b5ab24b6c69951", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_MZJEkR9JaEyLTxRf", + "ray.task_id": "16310a0f0a45af5ca703debb0a7c88156ee0838601000000", + "ray.worker_id": "aa57a10c2ebca86dc84acdc592b944b64b2a68489a9c2b64c2f9de2d", "span.kind": "consumer" }, "metrics": { @@ -195,8 +195,8 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 49042, - "start": 1764326801862827092 + "duration": 48958, + "start": 1764597968362880502 }, { "name": "exec entrypoint", @@ -210,13 +210,13 @@ "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "5d7b8f890a255711f2ceb2b401000000", + "ray.actor_id": "a703debb0a7c88156ee0838601000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", - "ray.task_id": "16310a0f0a45af5c5d7b8f890a255711f2ceb2b401000000", - "ray.worker_id": "08b10fdcbdb3706b74070824f41e97ce7ce7fb7f98b5ab24b6c69951", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_MZJEkR9JaEyLTxRf", + "ray.task_id": "16310a0f0a45af5ca703debb0a7c88156ee0838601000000", + "ray.worker_id": "aa57a10c2ebca86dc84acdc592b944b64b2a68489a9c2b64c2f9de2d", "span.kind": "consumer" }, "metrics": { @@ -226,84 +226,6 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 448708, - "start": 1764326801863071426 - }, - { - "name": "actor_method.submit", - "service": "unnamed.ray.job", - "resource": "Counter.increment.remote", - "trace_id": 0, - "span_id": 7, - "parent_id": 4, - "type": "ray", - "error": 0, - "meta": { - "_dd.base_service": "project", - "_dd.p.tid": "69297d9000000000", - "_dd.parent_id": "1ef55996279e487b", - "component": "ray", - "language": "python", - "ray.hostname": "docker-desktop", - "ray.job_id": "02000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", - "ray.worker_id": "02000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "1650ca0c39a544418fd27eaba368aa5e", - "span.kind": "producer", - "traceparent": "00-69297d90000000007b46d2532fc2b819-1ef55996279e487b-00", - "tracestate": "dd=p:1ef55996279e487b;s:0" - }, - "metrics": { - "_dd.ai_obs.enabled": 1.0, - "_dd.djm.enabled": 1.0, - "_dd.filter.kept": 1.0, - "_dd.measured": 1.0, - "_dd.top_level": 1.0, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 2.0, - "process_id": 1844.0 - }, - "duration": 301292, - "start": 1764326802310894009 - }, - { - "name": "actor_method.execute", - "service": "unnamed.ray.job", - "resource": "Counter.increment", - "trace_id": 0, - "span_id": 9, - "parent_id": 7, - "type": "ray", - "error": 0, - "meta": { - "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69297d9000000000", - "_dd.parent_id": "40bcf0c7bb993b59", - "component": "ray", - "language": "python", - "ray.actor_id": "b9b2522c73101eb8596ed95a02000000", - "ray.hostname": "docker-desktop", - "ray.job_id": "02000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_rKH1fCFUMaVzSGMN", - "ray.task_id": "e082c90ab8422b00b9b2522c73101eb8596ed95a02000000", - "ray.worker_id": "079fd52b7ed068a3f5c0913d3d3b740ca854d0ac958e4c213859c1d1", - "runtime-id": "be7f7861b5f5452e8d33426a3682f6da", - "span.kind": "consumer", - "traceparent": "00-69297d90000000007b46d2532fc2b819-40bcf0c7bb993b59-00", - "tracestate": "dd=p:40bcf0c7bb993b59;s:0" - }, - "metrics": { - "_dd.ai_obs.enabled": 1.0, - "_dd.djm.enabled": 1.0, - "_dd.filter.kept": 1.0, - "_dd.measured": 1.0, - "_dd.top_level": 1.0, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 2.0, - "process_id": 1892.0 - }, - "duration": 197291, - "start": 1764326803137070760 - }]] + "duration": 436000, + "start": 1764597968363127210 + }]] diff --git a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_job_name_specified.json b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_job_name_specified.json index d3d072f5855..66978c67864 100644 --- a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_job_name_specified.json +++ b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_job_name_specified.json @@ -11,7 +11,7 @@ "meta": { "_dd.base_service": "ray.dashboard", "_dd.p.dm": "-0", - "_dd.p.tid": "69297d9700000000", + "_dd.p.tid": "692da0d600000000", "component": "ray", "language": "python", "ray.entrypoint": "python /service.py", @@ -20,10 +20,10 @@ "ray.job.metadata.job_name": "my_model", "ray.job.status": "SUCCEEDED", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_vEnV5U2jQeTNiXNv", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "81f567481c4e41dc8f1b56ecf05d2973" + "runtime-id": "5f45deb5246d4c468a5bc92fb40b3998" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -33,10 +33,10 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1599.0 + "process_id": 1092.0 }, - "duration": 2987849876, - "start": 1764326807461016012 + "duration": 3823965543, + "start": 1764597974090235880 }, { "name": "ray.job.submit", @@ -53,8 +53,8 @@ "ray.hostname": "docker-desktop", "ray.job.submit_status": "success", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_vEnV5U2jQeTNiXNv", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", "span.kind": "producer" }, @@ -65,8 +65,8 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 56250, - "start": 1764326807461274095 + "duration": 56750, + "start": 1764597974090492671 }, { "name": "actor_method.submit", @@ -82,7 +82,7 @@ "component": "ray", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", "span.kind": "producer" }, @@ -93,8 +93,8 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 779417, - "start": 1764326807466330845 + "duration": 398417, + "start": 1764597974094428213 }, { "name": "actor_method.execute", @@ -107,21 +107,21 @@ "error": 0, "meta": { "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69297d9700000000", - "_dd.parent_id": "ef4b527279a09cfc", + "_dd.p.tid": "692da0d600000000", + "_dd.parent_id": "ba255f2302160faa", "component": "ray", "language": "python", - "ray.actor_id": "188ec803f749feb4a24a286901000000", + "ray.actor_id": "8541d17531033cc6dcc98ac901000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", - "ray.task_id": "82891771158d68c1188ec803f749feb4a24a286901000000", - "ray.worker_id": "73fd7ee2c79bd5108091d730a28b51046985a81f470fdc733ff23bcc", - "runtime-id": "c209b08c9fa04906971e43a2489a7118", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_vEnV5U2jQeTNiXNv", + "ray.task_id": "f91b78d7db9a65938541d17531033cc6dcc98ac901000000", + "ray.worker_id": "6fbf76c4a03387b3eaddf1693602f90a427e8a445c8d83db54cf437d", + "runtime-id": "dd1036b260354bbfbda54c0eef1e3999", "span.kind": "consumer", - "traceparent": "00-69297d970000000065054211638fe87d-ef4b527279a09cfc-00", - "tracestate": "dd=p:ef4b527279a09cfc" + "traceparent": "00-692da0d600000000e697be2587830d15-ba255f2302160faa-00", + "tracestate": "dd=p:ba255f2302160faa" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -131,10 +131,10 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1982.0 + "process_id": 1457.0 }, - "duration": 1929833376, - "start": 1764326808436916137 + "duration": 2031441918, + "start": 1764597974894888755 }, { "name": "actor_method.execute", @@ -148,13 +148,13 @@ "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "188ec803f749feb4a24a286901000000", + "ray.actor_id": "8541d17531033cc6dcc98ac901000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", - "ray.task_id": "82891771158d68c1188ec803f749feb4a24a286901000000", - "ray.worker_id": "73fd7ee2c79bd5108091d730a28b51046985a81f470fdc733ff23bcc", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_vEnV5U2jQeTNiXNv", + "ray.task_id": "f91b78d7db9a65938541d17531033cc6dcc98ac901000000", + "ray.worker_id": "6fbf76c4a03387b3eaddf1693602f90a427e8a445c8d83db54cf437d", "span.kind": "consumer" }, "metrics": { @@ -164,28 +164,28 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 1392625, - "start": 1764326808438404971 + "duration": 1243875, + "start": 1764597974896370047 }, { "name": "actor_method.execute", "service": "my_model", "resource": "JobSupervisor._get_driver_runtime_env", "trace_id": 0, - "span_id": 8, + "span_id": 7, "parent_id": 5, "type": "ray", "error": 0, "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "188ec803f749feb4a24a286901000000", + "ray.actor_id": "8541d17531033cc6dcc98ac901000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", - "ray.task_id": "82891771158d68c1188ec803f749feb4a24a286901000000", - "ray.worker_id": "73fd7ee2c79bd5108091d730a28b51046985a81f470fdc733ff23bcc", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_vEnV5U2jQeTNiXNv", + "ray.task_id": "f91b78d7db9a65938541d17531033cc6dcc98ac901000000", + "ray.worker_id": "6fbf76c4a03387b3eaddf1693602f90a427e8a445c8d83db54cf437d", "span.kind": "consumer" }, "metrics": { @@ -195,8 +195,8 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 50750, - "start": 1764326808439718179 + "duration": 42209, + "start": 1764597974897551005 }, { "name": "exec entrypoint", @@ -210,13 +210,13 @@ "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "188ec803f749feb4a24a286901000000", + "ray.actor_id": "8541d17531033cc6dcc98ac901000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", - "ray.task_id": "82891771158d68c1188ec803f749feb4a24a286901000000", - "ray.worker_id": "73fd7ee2c79bd5108091d730a28b51046985a81f470fdc733ff23bcc", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_vEnV5U2jQeTNiXNv", + "ray.task_id": "f91b78d7db9a65938541d17531033cc6dcc98ac901000000", + "ray.worker_id": "6fbf76c4a03387b3eaddf1693602f90a427e8a445c8d83db54cf437d", "span.kind": "consumer" }, "metrics": { @@ -226,82 +226,6 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 417667, - "start": 1764326808439976179 - }, - { - "name": "task.submit", - "service": "my_model", - "resource": "__main__.add_one.remote", - "trace_id": 0, - "span_id": 7, - "parent_id": 4, - "type": "ray", - "error": 0, - "meta": { - "_dd.base_service": "project", - "_dd.p.tid": "69297d9700000000", - "_dd.parent_id": "e4ea4e5b0eb7e1f6", - "component": "ray", - "language": "python", - "ray.hostname": "docker-desktop", - "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", - "ray.task.submit_status": "success", - "runtime-id": "799d236cda0040bf9119c0ab230d0b43", - "span.kind": "producer", - "traceparent": "00-69297d970000000065054211638fe87d-e4ea4e5b0eb7e1f6-00", - "tracestate": "dd=p:e4ea4e5b0eb7e1f6;s:0" - }, - "metrics": { - "_dd.ai_obs.enabled": 1.0, - "_dd.djm.enabled": 1.0, - "_dd.filter.kept": 1.0, - "_dd.measured": 1.0, - "_dd.top_level": 1.0, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 2.0, - "process_id": 2039.0 - }, - "duration": 51639750, - "start": 1764326808835367429 - }, - { - "name": "task.execute", - "service": "my_model", - "resource": "__main__.add_one", - "trace_id": 0, - "span_id": 9, - "parent_id": 7, - "type": "ray", - "error": 0, - "meta": { - "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69297d9700000000", - "_dd.parent_id": "a39c9bc39cce2b3e", - "component": "ray", - "language": "python", - "ray.hostname": "docker-desktop", - "ray.job_id": "03000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_q3MbATsz1DbYKRmm", - "ray.task.status": "success", - "ray.task_id": "fa31b89f94899135ffffffffffffffffffffffff03000000", - "ray.worker_id": "d5b3492c212cb35721dab353aa08670ed0020a8b262672d6ab6dc482", - "runtime-id": "05f200c2a93e4003bb5204474396476d", - "span.kind": "consumer", - "traceparent": "00-69297d970000000065054211638fe87d-a39c9bc39cce2b3e-00", - "tracestate": "dd=p:a39c9bc39cce2b3e;s:0" - }, - "metrics": { - "_dd.ai_obs.enabled": 1.0, - "_dd.djm.enabled": 1.0, - "_dd.filter.kept": 1.0, - "_dd.measured": 1.0, - "_dd.top_level": 1.0, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 2.0, - "process_id": 2087.0 - }, - "duration": 217875, - "start": 1764326809733088096 - }]] + "duration": 444416, + "start": 1764597974897782089 + }]] diff --git a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json index 0e51247d8e2..11e87b98e3e 100644 --- a/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json +++ b/tests/snapshots/tests.contrib.ray.test_ray.TestRayWithoutInit.test_task_without_init.json @@ -11,7 +11,7 @@ "meta": { "_dd.base_service": "ray.dashboard", "_dd.p.dm": "-0", - "_dd.p.tid": "69297d9d00000000", + "_dd.p.tid": "692da0dc00000000", "component": "ray", "language": "python", "ray.entrypoint": "python /task_without_init.py", @@ -20,10 +20,10 @@ "ray.job.metadata.foo": "bar", "ray.job.status": "SUCCEEDED", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_TcGEvgnfqQtHXEfa", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "81f567481c4e41dc8f1b56ecf05d2973" + "runtime-id": "5f45deb5246d4c468a5bc92fb40b3998" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -33,10 +33,10 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 1599.0 + "process_id": 1092.0 }, - "duration": 3928049627, - "start": 1764326813998606834 + "duration": 2776646210, + "start": 1764597980642195258 }, { "name": "ray.job.submit", @@ -53,8 +53,8 @@ "ray.hostname": "docker-desktop", "ray.job.submit_status": "success", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_TcGEvgnfqQtHXEfa", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", "span.kind": "producer" }, @@ -65,8 +65,8 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 62292, - "start": 1764326813998850042 + "duration": 68250, + "start": 1764597980642450258 }, { "name": "actor_method.submit", @@ -82,7 +82,7 @@ "component": "ray", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff", "span.kind": "producer" }, @@ -93,8 +93,8 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 387917, - "start": 1764326814002223584 + "duration": 384125, + "start": 1764597980645615008 }, { "name": "actor_method.execute", @@ -107,21 +107,21 @@ "error": 0, "meta": { "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69297d9d00000000", - "_dd.parent_id": "d90560ba034420ae", + "_dd.p.tid": "692da0dc00000000", + "_dd.parent_id": "a1a2fa0b2b7d5b09", "component": "ray", "language": "python", - "ray.actor_id": "690dcee46cf26e93ee89e59001000000", + "ray.actor_id": "f8ef7001025bb93e40a01d4201000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", - "ray.task_id": "85748392bcd969cc690dcee46cf26e93ee89e59001000000", - "ray.worker_id": "75bc236cdf71c7b5a1a4fa31b673d7f14572c4437fbc95fcedec2fa5", - "runtime-id": "c5a28ebae0c24da5b0a6f427ed8db095", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_TcGEvgnfqQtHXEfa", + "ray.task_id": "85748392bcd969ccf8ef7001025bb93e40a01d4201000000", + "ray.worker_id": "b6cb61f1e321a6f15f91dbb59dc6e926653a5f30133128b1b3f41392", + "runtime-id": "464d2607626f489ca30f273d3c521cd7", "span.kind": "consumer", - "traceparent": "00-69297d9d00000000bf349262b273a9f9-d90560ba034420ae-00", - "tracestate": "dd=p:d90560ba034420ae" + "traceparent": "00-692da0dc0000000098f4d29e4d961e0e-a1a2fa0b2b7d5b09-00", + "tracestate": "dd=p:a1a2fa0b2b7d5b09" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -131,10 +131,10 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 2153.0 + "process_id": 1620.0 }, - "duration": 2042603918, - "start": 1764326814908696334 + "duration": 1935175209, + "start": 1764597981411453467 }, { "name": "actor_method.execute", @@ -148,13 +148,13 @@ "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "690dcee46cf26e93ee89e59001000000", + "ray.actor_id": "f8ef7001025bb93e40a01d4201000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", - "ray.task_id": "85748392bcd969cc690dcee46cf26e93ee89e59001000000", - "ray.worker_id": "75bc236cdf71c7b5a1a4fa31b673d7f14572c4437fbc95fcedec2fa5", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_TcGEvgnfqQtHXEfa", + "ray.task_id": "85748392bcd969ccf8ef7001025bb93e40a01d4201000000", + "ray.worker_id": "b6cb61f1e321a6f15f91dbb59dc6e926653a5f30133128b1b3f41392", "span.kind": "consumer" }, "metrics": { @@ -164,28 +164,28 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 1157500, - "start": 1764326814910158876 + "duration": 1503000, + "start": 1764597981413220383 }, { "name": "actor_method.execute", "service": "unnamed.ray.job", "resource": "JobSupervisor._get_driver_runtime_env", "trace_id": 0, - "span_id": 11, + "span_id": 10, "parent_id": 5, "type": "ray", "error": 0, "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "690dcee46cf26e93ee89e59001000000", + "ray.actor_id": "f8ef7001025bb93e40a01d4201000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", - "ray.task_id": "85748392bcd969cc690dcee46cf26e93ee89e59001000000", - "ray.worker_id": "75bc236cdf71c7b5a1a4fa31b673d7f14572c4437fbc95fcedec2fa5", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_TcGEvgnfqQtHXEfa", + "ray.task_id": "85748392bcd969ccf8ef7001025bb93e40a01d4201000000", + "ray.worker_id": "b6cb61f1e321a6f15f91dbb59dc6e926653a5f30133128b1b3f41392", "span.kind": "consumer" }, "metrics": { @@ -195,8 +195,8 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 42750, - "start": 1764326814911251418 + "duration": 48625, + "start": 1764597981414645467 }, { "name": "exec entrypoint", @@ -210,13 +210,13 @@ "meta": { "_dd.base_service": "ray._private.workers", "component": "ray", - "ray.actor_id": "690dcee46cf26e93ee89e59001000000", + "ray.actor_id": "f8ef7001025bb93e40a01d4201000000", "ray.hostname": "docker-desktop", "ray.job_id": "01000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", - "ray.task_id": "85748392bcd969cc690dcee46cf26e93ee89e59001000000", - "ray.worker_id": "75bc236cdf71c7b5a1a4fa31b673d7f14572c4437fbc95fcedec2fa5", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_TcGEvgnfqQtHXEfa", + "ray.task_id": "85748392bcd969ccf8ef7001025bb93e40a01d4201000000", + "ray.worker_id": "b6cb61f1e321a6f15f91dbb59dc6e926653a5f30133128b1b3f41392", "span.kind": "consumer" }, "metrics": { @@ -226,8 +226,8 @@ "_dd.measured": 1.0, "_sampling_priority_v1": 2.0 }, - "duration": 401791, - "start": 1764326814911467793 + "duration": 503125, + "start": 1764597981414937842 }, { "name": "task.submit", @@ -240,96 +240,20 @@ "error": 0, "meta": { "_dd.base_service": "project", - "_dd.p.tid": "69297d9d00000000", - "_dd.parent_id": "c9a7f2428679c895", - "component": "ray", - "language": "python", - "ray.hostname": "docker-desktop", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", - "ray.task.submit_status": "success", - "runtime-id": "3f5cf2f03db844538d5bdfb2f5aca662", - "span.kind": "producer", - "traceparent": "00-69297d9d00000000bf349262b273a9f9-c9a7f2428679c895-00", - "tracestate": "dd=p:c9a7f2428679c895;s:0" - }, - "metrics": { - "_dd.ai_obs.enabled": 1.0, - "_dd.djm.enabled": 1.0, - "_dd.filter.kept": 1.0, - "_dd.measured": 1.0, - "_dd.top_level": 1.0, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 2.0, - "process_id": 2209.0 - }, - "duration": 46862458, - "start": 1764326815281651293 - }, - { - "name": "task.execute", - "service": "unnamed.ray.job", - "resource": "__main__.add_one", - "trace_id": 0, - "span_id": 12, - "parent_id": 7, - "type": "ray", - "error": 0, - "meta": { - "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69297d9d00000000", - "_dd.parent_id": "c9020337cd482b77", - "component": "ray", - "language": "python", - "ray.hostname": "docker-desktop", - "ray.job_id": "04000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", - "ray.task.status": "success", - "ray.task_id": "ca80f3a8e8ba2e50ffffffffffffffffffffffff04000000", - "ray.worker_id": "92ca8487cabc419e929466bc527769c1d49f235d58a0cde361b95018", - "runtime-id": "51b38c234ba24001a5c0360c6ddf2a6e", - "span.kind": "consumer", - "traceparent": "00-69297d9d00000000bf349262b273a9f9-c9020337cd482b77-00", - "tracestate": "dd=p:c9020337cd482b77;s:0" - }, - "metrics": { - "_dd.ai_obs.enabled": 1.0, - "_dd.djm.enabled": 1.0, - "_dd.filter.kept": 1.0, - "_dd.measured": 1.0, - "_dd.top_level": 1.0, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 2.0, - "process_id": 2257.0 - }, - "duration": 233875, - "start": 1764326816172440293 - }, - { - "name": "task.submit", - "service": "unnamed.ray.job", - "resource": "__main__.add_one.remote", - "trace_id": 0, - "span_id": 8, - "parent_id": 4, - "type": "ray", - "error": 0, - "meta": { - "_dd.base_service": "project", - "_dd.p.tid": "69297d9d00000000", - "_dd.parent_id": "c9a7f2428679c895", + "_dd.p.tid": "692da0dc00000000", + "_dd.parent_id": "eba90cfba438a54a", "component": "ray", "language": "python", "ray.hostname": "docker-desktop", "ray.job_id": "04000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_TcGEvgnfqQtHXEfa", "ray.task.submit_status": "success", "ray.worker_id": "04000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "3f5cf2f03db844538d5bdfb2f5aca662", + "runtime-id": "f56cbef8e0c34d289654bf86ddda3261", "span.kind": "producer", - "traceparent": "00-69297d9d00000000bf349262b273a9f9-c9a7f2428679c895-00", - "tracestate": "dd=p:c9a7f2428679c895;s:0" + "traceparent": "00-692da0dc0000000098f4d29e4d961e0e-eba90cfba438a54a-00", + "tracestate": "dd=p:eba90cfba438a54a;s:0" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -339,76 +263,36 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 2209.0 + "process_id": 1672.0 }, - "duration": 192875, - "start": 1764326815329576293 + "duration": 224792, + "start": 1764597981855136175 }, - { - "name": "task.execute", - "service": "unnamed.ray.job", - "resource": "__main__.add_one", - "trace_id": 0, - "span_id": 13, - "parent_id": 8, - "type": "ray", - "error": 0, - "meta": { - "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69297d9d00000000", - "_dd.parent_id": "09f655a6cbe5baf5", - "component": "ray", - "language": "python", - "ray.hostname": "docker-desktop", - "ray.job_id": "04000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", - "ray.task.status": "success", - "ray.task_id": "d8c1ad1586102897ffffffffffffffffffffffff04000000", - "ray.worker_id": "92ca8487cabc419e929466bc527769c1d49f235d58a0cde361b95018", - "runtime-id": "51b38c234ba24001a5c0360c6ddf2a6e", - "span.kind": "consumer", - "traceparent": "00-69297d9d00000000bf349262b273a9f9-09f655a6cbe5baf5-00", - "tracestate": "dd=p:09f655a6cbe5baf5;s:0" - }, - "metrics": { - "_dd.ai_obs.enabled": 1.0, - "_dd.djm.enabled": 1.0, - "_dd.filter.kept": 1.0, - "_dd.measured": 1.0, - "_dd.top_level": 1.0, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 2.0, - "process_id": 2257.0 - }, - "duration": 127666, - "start": 1764326816174502877 - }, { "name": "task.submit", "service": "unnamed.ray.job", "resource": "__main__.add_two.remote", "trace_id": 0, - "span_id": 9, + "span_id": 8, "parent_id": 4, "type": "ray", "error": 0, "meta": { "_dd.base_service": "project", - "_dd.p.tid": "69297d9d00000000", - "_dd.parent_id": "c9a7f2428679c895", + "_dd.p.tid": "692da0dc00000000", + "_dd.parent_id": "eba90cfba438a54a", "component": "ray", "language": "python", "ray.hostname": "docker-desktop", "ray.job_id": "04000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_TcGEvgnfqQtHXEfa", "ray.task.submit_status": "success", "ray.worker_id": "04000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "3f5cf2f03db844538d5bdfb2f5aca662", + "runtime-id": "f56cbef8e0c34d289654bf86ddda3261", "span.kind": "producer", - "traceparent": "00-69297d9d00000000bf349262b273a9f9-c9a7f2428679c895-00", - "tracestate": "dd=p:c9a7f2428679c895;s:0" + "traceparent": "00-692da0dc0000000098f4d29e4d961e0e-eba90cfba438a54a-00", + "tracestate": "dd=p:eba90cfba438a54a;s:0" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -418,37 +302,37 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 2209.0 + "process_id": 1672.0 }, - "duration": 1140167, - "start": 1764326816175653460 + "duration": 1352333, + "start": 1764597982512572176 }, { "name": "task.execute", "service": "unnamed.ray.job", "resource": "__main__.add_two", "trace_id": 0, - "span_id": 14, - "parent_id": 9, + "span_id": 11, + "parent_id": 8, "type": "ray", "error": 0, "meta": { "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69297d9d00000000", - "_dd.parent_id": "9a61c082c74ff91f", + "_dd.p.tid": "692da0dc00000000", + "_dd.parent_id": "3e0e099ddbc6fccd", "component": "ray", "language": "python", "ray.hostname": "docker-desktop", "ray.job_id": "04000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_TcGEvgnfqQtHXEfa", "ray.task.status": "success", "ray.task_id": "5da13aa9e2150759ffffffffffffffffffffffff04000000", - "ray.worker_id": "92ca8487cabc419e929466bc527769c1d49f235d58a0cde361b95018", - "runtime-id": "51b38c234ba24001a5c0360c6ddf2a6e", + "ray.worker_id": "c64b9baba4bfa6e75b826504d2a4e551a49f2069628f6bb8363b624d", + "runtime-id": "0a59a034188d4817964027f013d3df55", "span.kind": "consumer", - "traceparent": "00-69297d9d00000000bf349262b273a9f9-9a61c082c74ff91f-00", - "tracestate": "dd=p:9a61c082c74ff91f;s:0" + "traceparent": "00-692da0dc0000000098f4d29e4d961e0e-3e0e099ddbc6fccd-00", + "tracestate": "dd=p:3e0e099ddbc6fccd;s:0" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -458,36 +342,36 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 2257.0 + "process_id": 1720.0 }, - "duration": 148125, - "start": 1764326816178042418 + "duration": 236959, + "start": 1764597982515217092 }, { "name": "task.submit", "service": "unnamed.ray.job", "resource": "__main__.add_two.remote", "trace_id": 0, - "span_id": 10, + "span_id": 9, "parent_id": 4, "type": "ray", "error": 0, "meta": { "_dd.base_service": "project", - "_dd.p.tid": "69297d9d00000000", - "_dd.parent_id": "c9a7f2428679c895", + "_dd.p.tid": "692da0dc00000000", + "_dd.parent_id": "eba90cfba438a54a", "component": "ray", "language": "python", "ray.hostname": "docker-desktop", "ray.job_id": "04000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_TcGEvgnfqQtHXEfa", "ray.task.submit_status": "success", "ray.worker_id": "04000000ffffffffffffffffffffffffffffffffffffffffffffffff", - "runtime-id": "3f5cf2f03db844538d5bdfb2f5aca662", + "runtime-id": "f56cbef8e0c34d289654bf86ddda3261", "span.kind": "producer", - "traceparent": "00-69297d9d00000000bf349262b273a9f9-c9a7f2428679c895-00", - "tracestate": "dd=p:c9a7f2428679c895;s:0" + "traceparent": "00-692da0dc0000000098f4d29e4d961e0e-eba90cfba438a54a-00", + "tracestate": "dd=p:eba90cfba438a54a;s:0" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -497,37 +381,37 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 2209.0 + "process_id": 1672.0 }, - "duration": 150042, - "start": 1764326816177453460 + "duration": 180500, + "start": 1764597982514746342 }, { "name": "task.execute", "service": "unnamed.ray.job", "resource": "__main__.add_two", "trace_id": 0, - "span_id": 15, - "parent_id": 10, + "span_id": 12, + "parent_id": 9, "type": "ray", "error": 0, "meta": { "_dd.base_service": "ray._private.workers", - "_dd.p.tid": "69297d9d00000000", - "_dd.parent_id": "b9042c2e4e83c5b7", + "_dd.p.tid": "692da0dc00000000", + "_dd.parent_id": "2d2cf71bb6f9cf2f", "component": "ray", "language": "python", "ray.hostname": "docker-desktop", "ray.job_id": "04000000", - "ray.node_id": "3aecb6de3ac0f5a4c5fa788ae44ac5eea549c8aa4180ca4f84e8d276", - "ray.submission_id": "raysubmit_XDxYgKunFquhcwaf", + "ray.node_id": "a71b53315c6cad705c8a184acd7f454419e389423a645f0e2e365d24", + "ray.submission_id": "raysubmit_TcGEvgnfqQtHXEfa", "ray.task.status": "success", "ray.task_id": "011ae9488efe4e25ffffffffffffffffffffffff04000000", - "ray.worker_id": "92ca8487cabc419e929466bc527769c1d49f235d58a0cde361b95018", - "runtime-id": "51b38c234ba24001a5c0360c6ddf2a6e", + "ray.worker_id": "c64b9baba4bfa6e75b826504d2a4e551a49f2069628f6bb8363b624d", + "runtime-id": "0a59a034188d4817964027f013d3df55", "span.kind": "consumer", - "traceparent": "00-69297d9d00000000bf349262b273a9f9-b9042c2e4e83c5b7-00", - "tracestate": "dd=p:b9042c2e4e83c5b7;s:0" + "traceparent": "00-692da0dc0000000098f4d29e4d961e0e-2d2cf71bb6f9cf2f-00", + "tracestate": "dd=p:2d2cf71bb6f9cf2f;s:0" }, "metrics": { "_dd.ai_obs.enabled": 1.0, @@ -537,8 +421,8 @@ "_dd.top_level": 1.0, "_dd.tracer_kr": 1.0, "_sampling_priority_v1": 2.0, - "process_id": 2257.0 + "process_id": 1720.0 }, - "duration": 106583, - "start": 1764326816179173585 + "duration": 144083, + "start": 1764597982516926134 }]] From 2aac010abd6ed3670cc340863091865b054582b4 Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Mon, 1 Dec 2025 15:22:03 +0100 Subject: [PATCH 7/8] add releasenote --- .../notes/fix-ray-without-init-5e21fcf5117afd0c.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 releasenotes/notes/fix-ray-without-init-5e21fcf5117afd0c.yaml diff --git a/releasenotes/notes/fix-ray-without-init-5e21fcf5117afd0c.yaml b/releasenotes/notes/fix-ray-without-init-5e21fcf5117afd0c.yaml new file mode 100644 index 00000000000..2b44bf6ba5b --- /dev/null +++ b/releasenotes/notes/fix-ray-without-init-5e21fcf5117afd0c.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + ray: This fix resolves an issue where Ray jobs that did not explicitly call ``ray.init()`` at the top of their scripts + were not properly instrumented, resulting in incomplete traces. + To ensure full tracing capabilities, use ``ddtrace-run`` when starting your Ray cluster: + ``DD_PATCH_MODULES="ray:true,aiohttp:false,grpc:false,requests:false" ddtrace-run ray start --head``. From 9d673c1e7304ebcecac38524d5d54e5dd3143085 Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Mon, 1 Dec 2025 15:39:09 +0100 Subject: [PATCH 8/8] deactivate cross_language tasks --- ddtrace/contrib/internal/ray/patch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ddtrace/contrib/internal/ray/patch.py b/ddtrace/contrib/internal/ray/patch.py index 3e3a3b24be4..f4dcb7f6f4c 100644 --- a/ddtrace/contrib/internal/ray/patch.py +++ b/ddtrace/contrib/internal/ray/patch.py @@ -149,7 +149,9 @@ def _wrap_task_execution(wrapped, *args, **kwargs): def traced_submit_task(wrapped, instance, args, kwargs): """Trace task submission, i.e the func.remote() call""" - if instance._function.__module__ in RAY_TASK_MODULE_DENYLIST: + + # Tracing doesn't work for cross lang yet. + if instance._function.__module__ in RAY_TASK_MODULE_DENYLIST or instance._is_cross_language: return wrapped(*args, **kwargs) if tracer.current_span() is None: