|
| 1 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +# All rights reserved. |
| 3 | +# |
| 4 | +# This source code is licensed under the BSD-style license found in the |
| 5 | +# LICENSE file in the root directory of this source tree. |
| 6 | + |
| 7 | +import logging |
| 8 | +import operator |
| 9 | + |
| 10 | +import torch |
| 11 | +from executorch.exir.dialects._ops import ops as exir_ops |
| 12 | +from executorch.exir.passes.constant_prop_pass import _nodes_written_in_place |
| 13 | +from torch.fx.passes.infra.pass_base import PassBase, PassResult |
| 14 | +from torch.utils import _pytree as pytree |
| 15 | + |
| 16 | + |
| 17 | +def _reads_only_shape(node: torch.fx.Node) -> bool: |
| 18 | + """full_like with a constant fill value reads only the shape and dtype of |
| 19 | + its input. With a static shape its result is a constant, and OpenVINO |
| 20 | + folds it into one too. |
| 21 | + """ |
| 22 | + if node.target != exir_ops.edge.aten.full_like.default: |
| 23 | + return False |
| 24 | + if any( |
| 25 | + isinstance(a, torch.fx.Node) for a in (*node.args[1:], *node.kwargs.values()) |
| 26 | + ): |
| 27 | + return False |
| 28 | + val = node.meta.get("val") |
| 29 | + return isinstance(val, torch.Tensor) and all(isinstance(d, int) for d in val.shape) |
| 30 | + |
| 31 | + |
| 32 | +def _inputs(node: torch.fx.Node) -> list: |
| 33 | + return [] if _reads_only_shape(node) else node.all_input_nodes |
| 34 | + |
| 35 | + |
| 36 | +def _on_cpu(value): |
| 37 | + if isinstance(value, torch.Tensor): |
| 38 | + return value.cpu() |
| 39 | + if isinstance(value, torch.device): |
| 40 | + return torch.device("cpu") |
| 41 | + return value |
| 42 | + |
| 43 | + |
| 44 | +def _constant_nodes(graph_module: torch.fx.GraphModule) -> set: |
| 45 | + """Nodes that depend only on constants. Graph order puts inputs first.""" |
| 46 | + constants = set() |
| 47 | + written = _nodes_written_in_place(graph_module) |
| 48 | + for node in graph_module.graph.nodes: |
| 49 | + # Only tensors: a higher-order op such as cond reads its subgraphs, |
| 50 | + # which may hold a random op, through get_attr nodes too. |
| 51 | + if node.op == "get_attr": |
| 52 | + value = operator.attrgetter(node.target)(graph_module) |
| 53 | + if isinstance(value, torch.Tensor) and node not in written: |
| 54 | + constants.add(node) |
| 55 | + elif node.op == "call_function" and ( |
| 56 | + _reads_only_shape(node) |
| 57 | + or ( |
| 58 | + torch.Tag.nondeterministic_seeded |
| 59 | + not in getattr(node.target, "tags", ()) |
| 60 | + and all(n in constants for n in node.all_input_nodes) |
| 61 | + ) |
| 62 | + ): |
| 63 | + constants.add(node) |
| 64 | + return constants |
| 65 | + |
| 66 | + |
| 67 | +def _evaluate( |
| 68 | + graph_module: torch.fx.GraphModule, target: torch.fx.Node |
| 69 | +) -> torch.Tensor: |
| 70 | + """Value of target, computing only what it reads, on the CPU.""" |
| 71 | + needed = set() |
| 72 | + stack: list = [target] |
| 73 | + while stack: |
| 74 | + node = stack.pop() |
| 75 | + if node not in needed: |
| 76 | + needed.add(node) |
| 77 | + stack.extend(_inputs(node)) |
| 78 | + order = [n for n in graph_module.graph.nodes if n in needed] |
| 79 | + last_use = {i: n for n in order for i in _inputs(n)} |
| 80 | + values = {} |
| 81 | + for node in order: |
| 82 | + if node.op == "get_attr": |
| 83 | + values[node] = operator.attrgetter(node.target)(graph_module) |
| 84 | + continue |
| 85 | + if _reads_only_shape(node): |
| 86 | + like = node.args[0].meta["val"] |
| 87 | + args = ( |
| 88 | + torch.empty(like.shape, dtype=like.dtype, device="cpu"), |
| 89 | + *node.args[1:], |
| 90 | + ) |
| 91 | + kwargs = dict(node.kwargs) |
| 92 | + else: |
| 93 | + args, kwargs = torch.fx.node.map_arg( |
| 94 | + (node.args, node.kwargs), lambda n: values[n] |
| 95 | + ) |
| 96 | + # The values are the same on any device, and lowering for OpenVINO |
| 97 | + # must not need the GPU a constant was made for. |
| 98 | + args, kwargs = pytree.tree_map(_on_cpu, (args, kwargs)) |
| 99 | + if kwargs.get("pin_memory"): |
| 100 | + kwargs = {**kwargs, "pin_memory": False} |
| 101 | + values[node] = node.target(*args, **kwargs) |
| 102 | + for i in _inputs(node): |
| 103 | + if last_use[i] is node: |
| 104 | + del values[i] |
| 105 | + return values[target] |
| 106 | + |
| 107 | + |
| 108 | +class FoldConstantCumsumPass(PassBase): |
| 109 | + """Replace an integer cumsum that reads only constants with its result. |
| 110 | +
|
| 111 | + With an integer cumsum over a constant, OpenVINO's CPU plugin (2025.1 to |
| 112 | + 2026.4) gives wrong results from the second inference on: masks built from |
| 113 | + it keep only what earlier calls allowed. pi0 builds its attention masks |
| 114 | + this way. The OpenVINO nightly no longer does this, so the pass can go once |
| 115 | + the oldest supported OpenVINO release has the fix. Only these cumsums are |
| 116 | + folded; constants that feed no such cumsum, such as compressed weights, are |
| 117 | + left as they are. Runs on an unlifted module, where constants are get_attr |
| 118 | + nodes; buffers the graph writes to are not constants, and full_like of an |
| 119 | + input with a static shape is one. |
| 120 | + """ |
| 121 | + |
| 122 | + def call(self, graph_module: torch.fx.GraphModule) -> PassResult: |
| 123 | + graph = graph_module.graph |
| 124 | + constants = _constant_nodes(graph_module) |
| 125 | + cumsums = [ |
| 126 | + n |
| 127 | + for n in graph.nodes |
| 128 | + if n in constants |
| 129 | + and n.target == exir_ops.edge.aten.cumsum.default |
| 130 | + and not n.meta["val"].dtype.is_floating_point |
| 131 | + and not n.meta["val"].dtype.is_complex |
| 132 | + ] |
| 133 | + modified = False |
| 134 | + # One at a time, so a cumsum that cannot be evaluated does not keep |
| 135 | + # the others from being folded. |
| 136 | + for node in cumsums: |
| 137 | + try: |
| 138 | + value = _evaluate(graph_module, node) |
| 139 | + except RuntimeError as e: |
| 140 | + # Some ops have modes the CPU lacks. |
| 141 | + logging.warning( |
| 142 | + "OpenVINO: could not fold %s, so on a CPU some OpenVINO " |
| 143 | + "releases give wrong results for it from the second " |
| 144 | + "inference on: %s", |
| 145 | + node.name, |
| 146 | + e, |
| 147 | + ) |
| 148 | + continue |
| 149 | + name = f"_folded_cumsum_{node.name}" |
| 150 | + while hasattr(graph_module, name): |
| 151 | + name += "_" |
| 152 | + graph_module.register_buffer(name, value) |
| 153 | + with graph.inserting_before(node): |
| 154 | + folded = graph.get_attr(name) |
| 155 | + folded.meta = node.meta.copy() |
| 156 | + node.replace_all_uses_with(folded) |
| 157 | + modified = True |
| 158 | + if modified: |
| 159 | + graph.eliminate_dead_code() |
| 160 | + graph_module.recompile() |
| 161 | + return PassResult(graph_module, modified) |
0 commit comments