Skip to content

Commit f85faca

Browse files
committed
add fold_exp
1 parent cebf97e commit f85faca

1 file changed

Lines changed: 142 additions & 0 deletions

File tree

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
from mlir import ir
2+
from mlir.dialects import arith, ext, math, transform
3+
from mlir.dialects.transform import DiagnosedSilenceableFailure
4+
5+
from lighthouse.dialects.transform.transform_ext import TransformExtensionDialect
6+
7+
8+
def _fastmath_allows_transform(op: ir.Operation) -> bool:
9+
"""
10+
True if `op`'s `fastmath` flags permit `exp(a)/exp(b)` -> `exp(a-b)`.
11+
12+
The rewrite is not IEEE-preserving: it changes the rounding of the two
13+
intermediate exponentials into a single one, and it turns a division into a
14+
subtraction. `reassoc` is what licenses that regrouping, and the flag set
15+
also has to cover the values the identity ignores -- `exp(a)/exp(b)` is
16+
finite when both exponentials overflow or underflow together, whereas
17+
`exp(a-b)` is not (and vice versa), so `nnan`/`ninf` are required too.
18+
`fastmath<fast>` is the only flag set that provides all three.
19+
"""
20+
if "fastmath" not in op.attributes:
21+
return False
22+
return str(op.attributes["fastmath"]) == "#arith.fastmath<fast>"
23+
24+
25+
def _exp_operand(value: ir.Value) -> ir.Value | None:
26+
"""The argument of `value`'s defining `math.exp`, if it is a fast-math one."""
27+
producer = value.owner
28+
if not isinstance(producer, ir.Operation):
29+
producer = getattr(producer, "operation", None)
30+
if producer is None or producer.name != "math.exp":
31+
return None
32+
if not _fastmath_allows_transform(producer):
33+
return None
34+
return producer.operands[0]
35+
36+
37+
def _collect_divf_ops(root: ir.Operation) -> list[ir.Operation]:
38+
ops: list[ir.Operation] = []
39+
40+
def collect(op: ir.Operation) -> ir.WalkResult:
41+
if op.name == "arith.divf":
42+
ops.append(op)
43+
return ir.WalkResult.ADVANCE
44+
45+
root.walk(collect, ir.WalkOrder.PRE_ORDER)
46+
return ops
47+
48+
49+
def _rewrite_divf(div_op: ir.Operation, rewriter: transform.TransformRewriter) -> bool:
50+
if not _fastmath_allows_transform(div_op):
51+
return False
52+
53+
numerator = _exp_operand(div_op.operands[0])
54+
denominator = _exp_operand(div_op.operands[1])
55+
if numerator is None or denominator is None:
56+
return False
57+
58+
# exp(a)/exp(b) and exp(a-b) only agree when a and b -- hence a-b -- share the
59+
# result type; a mixed-precision chain is left alone.
60+
if numerator.type != denominator.type:
61+
return False
62+
if numerator.type != div_op.results[0].type:
63+
return False
64+
65+
fast = arith.FastMathFlags.fast
66+
with ir.InsertionPoint(div_op), div_op.location:
67+
difference = arith.SubFOp(numerator, denominator, fastmath=fast)
68+
replacement = math.ExpOp(difference.result, fastmath=fast)
69+
# The two original exps are left in place; they are dead only if nothing else
70+
# reads them, which DCE/CSE afterwards decides.
71+
rewriter.replace_op(div_op, [replacement.result])
72+
return True
73+
74+
75+
class FoldExpDivOp(TransformExtensionDialect.Operation, name="fold_exp_div"):
76+
"""
77+
Rewrite `exp(a) / exp(b)` into `exp(a - b)`.
78+
79+
Trades a transcendental and a divide for a subtract. Applies only when the
80+
`arith.divf` and both `math.exp` producers all carry `fastmath<fast>`; see
81+
`_fastmath_allows_transform` for why nothing weaker is accepted. Ops without
82+
the flag are skipped rather than rejected, so this is safe to run over a whole
83+
function.
84+
85+
The motivating case is the online-softmax rescale factor emitted by
86+
`transform.structured.fuse_dependant_reduction_ops`, which is
87+
`exp(-m_new) / exp(-m_old)`; this turns it into the single `exp(m_old - m_new)`
88+
a hand-written flash-attention kernel uses. Run `set_fastmath` first: the
89+
divide originates from a `linalg.elementwise`, which has no `fastmath`
90+
attribute to inherit.
91+
92+
Args:
93+
target: Handle to root ops to rewrite within (e.g. func.func).
94+
Returns:
95+
Handle to the (possibly) rewritten target roots.
96+
"""
97+
98+
target: ext.Operand[transform.AnyOpType]
99+
rewritten_ops: ext.Result[transform.AnyOpType[()]] = ext.infer_result()
100+
101+
@classmethod
102+
def attach_interface_impls(cls, context=None):
103+
cls.TransformOpInterfaceModel.attach(cls.OPERATION_NAME, context=context)
104+
cls.MemoryEffectsOpInterfaceModel.attach(cls.OPERATION_NAME, context=context)
105+
106+
class TransformOpInterfaceModel(transform.TransformOpInterface):
107+
@staticmethod
108+
def apply(
109+
op: "FoldExpDivOp",
110+
rewriter: transform.TransformRewriter,
111+
results: transform.TransformResults,
112+
state: transform.TransformState,
113+
) -> DiagnosedSilenceableFailure:
114+
targets = state.get_payload_ops(op.target)
115+
116+
for target in targets:
117+
for div_op in _collect_divf_ops(target):
118+
_rewrite_divf(div_op, rewriter)
119+
120+
results.set_ops(op.rewritten_ops, targets)
121+
return DiagnosedSilenceableFailure.Success
122+
123+
@staticmethod
124+
def allow_repeated_handle_operands(_op: "FoldExpDivOp") -> bool:
125+
return False
126+
127+
class MemoryEffectsOpInterfaceModel(ir.MemoryEffectsOpInterface):
128+
@staticmethod
129+
def get_effects(op: ir.Operation):
130+
return (
131+
transform.only_reads_handle(op.op_operands)
132+
+ transform.produces_handle(op.results)
133+
+ transform.modifies_payload()
134+
)
135+
136+
137+
def fold_exp_div(
138+
target: ir.Value[transform.AnyOpType],
139+
) -> ir.Value[transform.AnyOpType]:
140+
"""snake_case wrapper to create FoldExpDivOp."""
141+
op = FoldExpDivOp(target=target)
142+
return op.rewritten_ops

0 commit comments

Comments
 (0)