Skip to content

Commit 2a2e567

Browse files
committed
fix torch llama example
1 parent 3b9fbe8 commit 2a2e567

1 file changed

Lines changed: 7 additions & 3 deletions

File tree

‎examples/llama/ref_model.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -273,9 +273,13 @@ def forward(self, tokens: torch.Tensor, start_pos: int):
273273
# only for the new sequence. Thus, the matrix of scores is of size
274274
# (seqlen, cache_len + seqlen), and the only masked entries are (i, j) for
275275
# j > cache_len + i, since row i corresponds to token cache_len + i.
276-
mask = torch.hstack(
277-
[torch.zeros((seqlen, start_pos), device=tokens.device), mask]
278-
).type_as(h)
276+
# Skip the empty (seqlen, 0) prefix when start_pos == 0: torch-mlir
277+
# cannot lower the resulting zero-sized aten.empty_strided.
278+
if start_pos > 0:
279+
mask = torch.hstack(
280+
[torch.zeros((seqlen, start_pos), device=tokens.device), mask]
281+
)
282+
mask = mask.type_as(h)
279283

280284
for layer in self.layers:
281285
h = layer(h, start_pos, freqs_cis, mask)

0 commit comments

Comments
 (0)