Skip to content

Commit 24c5908

Browse files
authored
[utils][runner] Reduce host overheads (#301)
Reduces the host-side work done on every call of a jitted function, such as preparing its arguments and looking up the function. Previously, every call repeated this work, so the overhead grew with the number of arguments and with the number of calls already made. The setup is now done once and reused, which keeps the overhead per call low and stable. Adds a script to measure the host-side call overhead. Assisted-by: Claude
1 parent 0d9bf5c commit 24c5908

7 files changed

Lines changed: 346 additions & 39 deletions

File tree

‎lighthouse/execution/runner.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ def __init__(
6969
self.shared_libs = list(dict.fromkeys(shared_libs))
7070
self.opt_level = opt_level
7171
self.engine = self._get_engine()
72+
# The functions of the engine, looked up on first call.
73+
self._functions: dict[str, Callable] = {}
7274
self._configure_system()
7375

7476
def _configure_system(self):
@@ -211,7 +213,10 @@ def _execute_kernel(
211213
function_name = payload_function_name
212214

213215
# Now lookup and call the function
214-
func = self.engine.lookup(function_name)
216+
func = self._functions.get(function_name)
217+
if func is None:
218+
func = self.engine.lookup(function_name)
219+
self._functions[function_name] = func
215220
func(args)
216221

217222
# If an argument access callback is provided,

‎lighthouse/utils/memref.py‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,23 @@ def to_packed_args(args) -> ctypes.Array[ctypes.c_void_p]:
3232
"""
3333
Convert a list of memref descriptors and/or integers into packed ctype arguments.
3434
35+
The packed arguments keep the descriptors alive.
36+
3537
Args:
3638
args: A list of memref descriptors or integers.
3739
"""
38-
ctype_args = []
39-
for arg in args:
40-
if isinstance(arg, int):
41-
ctype_args.append(ctypes.pointer(ctypes.c_int64(arg)))
42-
else:
43-
ctype_args.append(to_ctype(arg))
44-
return get_packed_arg(ctype_args)
40+
args = tuple(args)
41+
# The storage of each argument: an integer, or a pointer to a descriptor.
42+
slots = (ctypes.c_int64 * len(args))(
43+
*[arg if isinstance(arg, int) else ctypes.addressof(arg) for arg in args]
44+
)
45+
base = ctypes.addressof(slots)
46+
size = ctypes.sizeof(ctypes.c_int64)
47+
packed_args = (ctypes.c_void_p * len(args))(
48+
*[base + i * size for i in range(len(args))]
49+
)
50+
# The callee only gets raw addresses.
51+
# Keep the storage of the arguments alive to prevent them from being
52+
# garbage collected.
53+
packed_args._keep_alive = (slots, args)
54+
return packed_args

‎lighthouse/utils/torch.py‎

Lines changed: 52 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import ctypes
2+
import struct
3+
from functools import cache
24

35
import torch
46
from mlir import ir
@@ -11,6 +13,19 @@
1113

1214
from . import memref as memref_utils
1315

16+
_DTYPE_TO_CTYPE = {
17+
torch.float16: F16,
18+
torch.bfloat16: BF16,
19+
torch.float32: ctypes.c_float,
20+
torch.float64: ctypes.c_double,
21+
torch.int8: ctypes.c_int8,
22+
torch.int16: ctypes.c_int16,
23+
torch.int32: ctypes.c_int32,
24+
torch.int64: ctypes.c_int64,
25+
torch.uint8: ctypes.c_uint8,
26+
torch.bool: ctypes.c_bool,
27+
}
28+
1429

1530
def torch_dtype_to_ctype(dtype: torch.dtype):
1631
"""
@@ -21,49 +36,55 @@ def torch_dtype_to_ctype(dtype: torch.dtype):
2136
Returns:
2237
Corresponding ctype
2338
"""
24-
_dtype_map = {
25-
torch.float16: F16,
26-
torch.bfloat16: BF16,
27-
torch.float32: ctypes.c_float,
28-
torch.float64: ctypes.c_double,
29-
torch.int8: ctypes.c_int8,
30-
torch.int16: ctypes.c_int16,
31-
torch.int32: ctypes.c_int32,
32-
torch.int64: ctypes.c_int64,
33-
torch.uint8: ctypes.c_uint8,
34-
torch.bool: ctypes.c_bool,
35-
}
36-
if dtype not in _dtype_map:
39+
if dtype not in _DTYPE_TO_CTYPE:
3740
raise ValueError(f"Unsupported dtype: {dtype}")
38-
return _dtype_map[dtype]
41+
return _DTYPE_TO_CTYPE[dtype]
42+
43+
44+
def _struct_format(field_type: type) -> str:
45+
"""The `struct` format of a ctypes field type."""
46+
if issubclass(field_type, ctypes.Array):
47+
return f"{field_type._length_}{_struct_format(field_type._type_)}"
48+
if issubclass(field_type, ctypes._Pointer):
49+
return "P"
50+
return field_type._type_
51+
52+
53+
@cache
54+
def _memref_descriptor(ndim: int, ctype: type) -> tuple[type, struct.Struct]:
55+
"""The memref descriptor class of a rank and element type, and its layout.
56+
57+
A new class per call would also make ctypes create new pointer types.
58+
"""
59+
if ndim == 0:
60+
descriptor = make_zero_d_memref_descriptor(ctype)
61+
else:
62+
descriptor = make_nd_memref_descriptor(ndim, ctype)
63+
layout = struct.Struct(
64+
"".join(_struct_format(field_type) for _, field_type in descriptor._fields_)
65+
)
66+
if layout.size != ctypes.sizeof(descriptor):
67+
raise RuntimeError(f"Unexpected memref descriptor layout: {layout.format}")
68+
return descriptor, layout
3969

4070

4171
def to_memref(input: torch.Tensor) -> ctypes.Structure:
4272
"""
4373
Convert a PyTorch tensor into a memref descriptor.
4474
75+
The descriptor does not keep the tensor alive.
76+
4577
Args:
4678
input: PyTorch tensor.
4779
"""
48-
ctp = torch_dtype_to_ctype(input.dtype)
49-
ndim = input.dim()
80+
descriptor, layout = _memref_descriptor(
81+
input.dim(), torch_dtype_to_ctype(input.dtype)
82+
)
5083
data_ptr = input.data_ptr()
51-
52-
if ndim == 0:
53-
x = make_zero_d_memref_descriptor(ctp)()
54-
x.allocated = data_ptr
55-
x.aligned = ctypes.cast(data_ptr, ctypes.POINTER(ctp))
56-
x.offset = ctypes.c_longlong(0)
57-
return x
58-
59-
x = make_nd_memref_descriptor(ndim, ctp)()
60-
x.allocated = data_ptr
61-
x.aligned = ctypes.cast(data_ptr, ctypes.POINTER(ctp))
62-
x.offset = ctypes.c_longlong(0)
63-
x.shape = (ctypes.c_longlong * ndim)(*input.shape)
6484
# PyTorch strides are in element units (unlike NumPy which uses bytes)
65-
x.strides = (ctypes.c_longlong * ndim)(*input.stride())
66-
return x
85+
return descriptor.from_buffer_copy(
86+
layout.pack(data_ptr, data_ptr, 0, *input.shape, *input.stride())
87+
)
6788

6889

6990
def to_packed_args(inputs: list[torch.Tensor]) -> ctypes.Array[ctypes.c_void_p]:
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/usr/bin/env python3
2+
"""Measure the host-side overhead of calling a JIT-compiled kernel.
3+
4+
Runs an empty kernel through lighthouse's Runner with PyTorch tensors, so all
5+
the measured time is host overhead: memref descriptor creation (to_memref),
6+
argument packing, function lookup and the call itself.
7+
"""
8+
9+
import argparse
10+
import statistics
11+
import time
12+
from collections.abc import Callable
13+
14+
import torch
15+
from mlir import ir
16+
from mlir.passmanager import PassManager
17+
18+
import lighthouse.dialects as lh_dialects
19+
from lighthouse.execution.runner import Runner
20+
from lighthouse.ingress.torch.compile import TorchMemoryManager
21+
from lighthouse.utils.memref import to_packed_args
22+
from lighthouse.utils.torch import to_memref
23+
24+
25+
def empty_kernel(num_args: int, rank: int) -> ir.Module:
26+
"""A function of `num_args` f32 memrefs of `rank` that does nothing."""
27+
shape = "x".join(["?"] * rank)
28+
args = ", ".join(f"%a{i}: memref<{shape}xf32>" for i in range(num_args))
29+
module = ir.Module.parse(f"func.func @main({args}) {{ return }}")
30+
Runner.make_function_callable(module, "main")
31+
PassManager.parse(
32+
"builtin.module(finalize-memref-to-llvm, convert-to-llvm, "
33+
"reconcile-unrealized-casts)"
34+
).run(module.operation)
35+
return module
36+
37+
38+
def time_us(fn: Callable[[], object], iters: int, repeats: int = 5) -> float:
39+
"""The median over `repeats` of the mean time per call of `fn`, in us."""
40+
for _ in range(min(iters, 1000)):
41+
fn()
42+
samples = []
43+
for _ in range(repeats):
44+
start = time.perf_counter_ns()
45+
for _ in range(iters):
46+
fn()
47+
samples.append((time.perf_counter_ns() - start) / iters / 1e3)
48+
return statistics.median(samples)
49+
50+
51+
def main() -> None:
52+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
53+
parser.add_argument("--iters", type=int, default=20000, help="calls per sample")
54+
parser.add_argument(
55+
"--num-args", type=int, nargs="+", default=[1, 3, 8], help="kernel arguments"
56+
)
57+
parser.add_argument(
58+
"--ranks", type=int, nargs="+", default=[1, 2, 4], help="argument ranks"
59+
)
60+
args = parser.parse_args()
61+
62+
print("Time per call in us; 'call' is the JIT call with prepared arguments.")
63+
header = ("args", "rank", "to_memref/arg", "to_packed_args", "execute", "call")
64+
print("".join(f"{name:>16}" for name in header))
65+
for num_args in args.num_args:
66+
for rank in args.ranks:
67+
with ir.Context(), ir.Location.unknown():
68+
lh_dialects.register_and_load()
69+
module = empty_kernel(num_args, rank)
70+
runner = Runner(module, mem_manager_cls=TorchMemoryManager)
71+
tensors = [torch.zeros([4] * rank) for _ in range(num_args)]
72+
descriptors = [to_memref(tensor) for tensor in tensors]
73+
packed_args = to_packed_args(descriptors)
74+
func = runner.engine.lookup("main")
75+
76+
times = (
77+
time_us(lambda: to_memref(tensors[0]), args.iters),
78+
time_us(lambda: to_packed_args(descriptors), args.iters),
79+
time_us(lambda: runner.execute("main", tensors), args.iters),
80+
time_us(lambda: func(packed_args), args.iters),
81+
)
82+
row = f"{num_args:>16}{rank:>16}"
83+
print(row + "".join(f"{t:>16.2f}" for t in times))
84+
85+
86+
if __name__ == "__main__":
87+
main()

‎test/execution/packed_args.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# RUN: %PYTHON %s | FileCheck %s
2+
3+
"""Tests for packing JIT function arguments."""
4+
5+
import ctypes
6+
import gc
7+
import weakref
8+
9+
import numpy as np
10+
from mlir.runtime.np_to_memref import get_ranked_memref_descriptor
11+
12+
from lighthouse.utils.memref import to_packed_args
13+
14+
array = np.arange(12, dtype=np.float32).reshape(3, 4)
15+
16+
# The descriptor is referenced by the packed arguments only.
17+
desc = get_ranked_memref_descriptor(array)
18+
desc_ref = weakref.ref(desc)
19+
packed = to_packed_args([desc, 7, -1])
20+
del desc
21+
gc.collect()
22+
# CHECK: descriptor alive: True
23+
print("descriptor alive:", desc_ref() is not None)
24+
25+
# Each packed argument points to the argument's storage: a pointer to the
26+
# descriptor, or the integer itself.
27+
desc_type = type(desc_ref())
28+
desc_address = ctypes.cast(packed[0], ctypes.POINTER(ctypes.c_void_p)).contents.value
29+
desc = desc_type.from_address(desc_address)
30+
# CHECK: descriptor shape: [3, 4] data: [0.0, 1.0, 2.0, 3.0]
31+
print(
32+
"descriptor shape:",
33+
list(desc.shape),
34+
"data:",
35+
[desc.aligned[i] for i in range(4)],
36+
)
37+
# CHECK: integers: 7 -1
38+
print(
39+
"integers:",
40+
ctypes.cast(packed[1], ctypes.POINTER(ctypes.c_int64)).contents.value,
41+
ctypes.cast(packed[2], ctypes.POINTER(ctypes.c_int64)).contents.value,
42+
)
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# RUN: %PYTHON %s | FileCheck %s
2+
3+
"""Tests that a Runner looks up each JIT function once."""
4+
5+
import numpy as np
6+
from mlir import ir
7+
from mlir.passmanager import PassManager
8+
9+
from lighthouse.execution.runner import Runner
10+
11+
12+
class CountingEngine:
13+
"""Forwards to an execution engine, counting function lookups."""
14+
15+
def __init__(self, engine):
16+
self.engine = engine
17+
self.lookups = 0
18+
19+
def lookup(self, name):
20+
self.lookups += 1
21+
return self.engine.lookup(name)
22+
23+
def __getattr__(self, name):
24+
return getattr(self.engine, name)
25+
26+
27+
with ir.Context(), ir.Location.unknown():
28+
module = ir.Module.parse(
29+
"""
30+
func.func @increment(%a: memref<4xf32>) {
31+
%c0 = arith.constant 0 : index
32+
%one = arith.constant 1.0 : f32
33+
%v = memref.load %a[%c0] : memref<4xf32>
34+
%s = arith.addf %v, %one : f32
35+
memref.store %s, %a[%c0] : memref<4xf32>
36+
return
37+
}
38+
"""
39+
)
40+
Runner.make_function_callable(module, "increment")
41+
PassManager.parse(
42+
"builtin.module(finalize-memref-to-llvm, convert-to-llvm, "
43+
"reconcile-unrealized-casts)"
44+
).run(module.operation)
45+
runner = Runner(module)
46+
47+
engine = runner.engine = CountingEngine(runner.engine)
48+
buffer = np.zeros(4, dtype=np.float32)
49+
for _ in range(3):
50+
runner.execute("increment", [buffer])
51+
52+
# CHECK: value: 3.0 lookups: 1
53+
print("value:", buffer[0], "lookups:", engine.lookups)
54+
55+
# A missing function still fails on every call.
56+
for _ in range(2):
57+
try:
58+
runner.execute("missing", [buffer])
59+
except Exception as error:
60+
# CHECK-COUNT-2: error: Unknown function missing
61+
print("error:", error)

0 commit comments

Comments
 (0)