|
| 1 | +from mlir import ir |
| 2 | +from mlir.dialects import arith, ext, math, transform |
| 3 | +from mlir.dialects.transform import DiagnosedSilenceableFailure |
| 4 | + |
| 5 | +from lighthouse.dialects.transform.transform_ext import TransformExtensionDialect |
| 6 | + |
| 7 | + |
| 8 | +def _fastmath_allows_transform(op: ir.Operation) -> bool: |
| 9 | + """ |
| 10 | + True if `op`'s `fastmath` flags permit `exp(a)/exp(b)` -> `exp(a-b)`. |
| 11 | +
|
| 12 | + The rewrite is not IEEE-preserving: it changes the rounding of the two |
| 13 | + intermediate exponentials into a single one, and it turns a division into a |
| 14 | + subtraction. `reassoc` is what licenses that regrouping, and the flag set |
| 15 | + also has to cover the values the identity ignores -- `exp(a)/exp(b)` is |
| 16 | + finite when both exponentials overflow or underflow together, whereas |
| 17 | + `exp(a-b)` is not (and vice versa), so `nnan`/`ninf` are required too. |
| 18 | + `fastmath<fast>` is the only flag set that provides all three. |
| 19 | + """ |
| 20 | + if "fastmath" not in op.attributes: |
| 21 | + return False |
| 22 | + return str(op.attributes["fastmath"]) == "#arith.fastmath<fast>" |
| 23 | + |
| 24 | + |
| 25 | +def _exp_operand(value: ir.Value) -> ir.Value | None: |
| 26 | + """The argument of `value`'s defining `math.exp`, if it is a fast-math one.""" |
| 27 | + producer = value.owner |
| 28 | + if not isinstance(producer, ir.Operation): |
| 29 | + producer = getattr(producer, "operation", None) |
| 30 | + if producer is None or producer.name != "math.exp": |
| 31 | + return None |
| 32 | + if not _fastmath_allows_transform(producer): |
| 33 | + return None |
| 34 | + return producer.operands[0] |
| 35 | + |
| 36 | + |
| 37 | +def _collect_divf_ops(root: ir.Operation) -> list[ir.Operation]: |
| 38 | + ops: list[ir.Operation] = [] |
| 39 | + |
| 40 | + def collect(op: ir.Operation) -> ir.WalkResult: |
| 41 | + if op.name == "arith.divf": |
| 42 | + ops.append(op) |
| 43 | + return ir.WalkResult.ADVANCE |
| 44 | + |
| 45 | + root.walk(collect, ir.WalkOrder.PRE_ORDER) |
| 46 | + return ops |
| 47 | + |
| 48 | + |
| 49 | +def _rewrite_divf(div_op: ir.Operation, rewriter: transform.TransformRewriter) -> bool: |
| 50 | + if not _fastmath_allows_transform(div_op): |
| 51 | + return False |
| 52 | + |
| 53 | + numerator = _exp_operand(div_op.operands[0]) |
| 54 | + denominator = _exp_operand(div_op.operands[1]) |
| 55 | + if numerator is None or denominator is None: |
| 56 | + return False |
| 57 | + |
| 58 | + # exp(a)/exp(b) and exp(a-b) only agree when a and b -- hence a-b -- share the |
| 59 | + # result type; a mixed-precision chain is left alone. |
| 60 | + if numerator.type != denominator.type: |
| 61 | + return False |
| 62 | + if numerator.type != div_op.results[0].type: |
| 63 | + return False |
| 64 | + |
| 65 | + fast = arith.FastMathFlags.fast |
| 66 | + with ir.InsertionPoint(div_op), div_op.location: |
| 67 | + difference = arith.SubFOp(numerator, denominator, fastmath=fast) |
| 68 | + replacement = math.ExpOp(difference.result, fastmath=fast) |
| 69 | + # The two original exps are left in place; they are dead only if nothing else |
| 70 | + # reads them, which DCE/CSE afterwards decides. |
| 71 | + rewriter.replace_op(div_op, [replacement.result]) |
| 72 | + return True |
| 73 | + |
| 74 | + |
| 75 | +class FoldExpDivOp(TransformExtensionDialect.Operation, name="fold_exp_div"): |
| 76 | + """ |
| 77 | + Rewrite `exp(a) / exp(b)` into `exp(a - b)`. |
| 78 | +
|
| 79 | + Trades a transcendental and a divide for a subtract. Applies only when the |
| 80 | + `arith.divf` and both `math.exp` producers all carry `fastmath<fast>`; see |
| 81 | + `_fastmath_allows_transform` for why nothing weaker is accepted. Ops without |
| 82 | + the flag are skipped rather than rejected, so this is safe to run over a whole |
| 83 | + function. |
| 84 | +
|
| 85 | + The motivating case is the online-softmax rescale factor emitted by |
| 86 | + `transform.structured.fuse_dependant_reduction_ops`, which is |
| 87 | + `exp(-m_new) / exp(-m_old)`; this turns it into the single `exp(m_old - m_new)` |
| 88 | + a hand-written flash-attention kernel uses. Run `set_fastmath` first: the |
| 89 | + divide originates from a `linalg.elementwise`, which has no `fastmath` |
| 90 | + attribute to inherit. |
| 91 | +
|
| 92 | + Args: |
| 93 | + target: Handle to root ops to rewrite within (e.g. func.func). |
| 94 | + Returns: |
| 95 | + Handle to the (possibly) rewritten target roots. |
| 96 | + """ |
| 97 | + |
| 98 | + target: ext.Operand[transform.AnyOpType] |
| 99 | + rewritten_ops: ext.Result[transform.AnyOpType[()]] = ext.infer_result() |
| 100 | + |
| 101 | + @classmethod |
| 102 | + def attach_interface_impls(cls, context=None): |
| 103 | + cls.TransformOpInterfaceModel.attach(cls.OPERATION_NAME, context=context) |
| 104 | + cls.MemoryEffectsOpInterfaceModel.attach(cls.OPERATION_NAME, context=context) |
| 105 | + |
| 106 | + class TransformOpInterfaceModel(transform.TransformOpInterface): |
| 107 | + @staticmethod |
| 108 | + def apply( |
| 109 | + op: "FoldExpDivOp", |
| 110 | + rewriter: transform.TransformRewriter, |
| 111 | + results: transform.TransformResults, |
| 112 | + state: transform.TransformState, |
| 113 | + ) -> DiagnosedSilenceableFailure: |
| 114 | + targets = state.get_payload_ops(op.target) |
| 115 | + |
| 116 | + for target in targets: |
| 117 | + for div_op in _collect_divf_ops(target): |
| 118 | + _rewrite_divf(div_op, rewriter) |
| 119 | + |
| 120 | + results.set_ops(op.rewritten_ops, targets) |
| 121 | + return DiagnosedSilenceableFailure.Success |
| 122 | + |
| 123 | + @staticmethod |
| 124 | + def allow_repeated_handle_operands(_op: "FoldExpDivOp") -> bool: |
| 125 | + return False |
| 126 | + |
| 127 | + class MemoryEffectsOpInterfaceModel(ir.MemoryEffectsOpInterface): |
| 128 | + @staticmethod |
| 129 | + def get_effects(op: ir.Operation): |
| 130 | + return ( |
| 131 | + transform.only_reads_handle(op.op_operands) |
| 132 | + + transform.produces_handle(op.results) |
| 133 | + + transform.modifies_payload() |
| 134 | + ) |
| 135 | + |
| 136 | + |
| 137 | +def fold_exp_div( |
| 138 | + target: ir.Value[transform.AnyOpType], |
| 139 | +) -> ir.Value[transform.AnyOpType]: |
| 140 | + """snake_case wrapper to create FoldExpDivOp.""" |
| 141 | + op = FoldExpDivOp(target=target) |
| 142 | + return op.rewritten_ops |
0 commit comments