From 1f6e62b950855f66e0da093d4dc8d40d9d547e97 Mon Sep 17 00:00:00 2001 From: ydshieh Date: Tue, 10 Nov 2026 15:24:13 +0100 Subject: [PATCH 1/2] Fix flaky ONNX export test for wav2vec2-style `ForPreTraining` models The codevector quantizer picks a code with a hard `argmax` in eval mode. On the random tiny models, two code logits can be within a few float32 ulps, so ONNX Runtime's rounding flips the choice and `codevector_perplexity` (a scalar) jumps far over the tolerance. Add `Wav2Vec2ConformerForPreTraining` and `UniSpeechSatForPreTraining` to `onnx.exactness` next to `Wav2Vec2ForPreTraining` and `UniSpeechForPreTraining`, and correct the reason (it is an argmax near-tie, not a Gumbel-softmax draw). Co-Authored-By: Claude Opus 5.5 --- tests/exporters/test_export.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/exporters/test_export.py b/tests/exporters/test_export.py index 459875d0d99a..30bc6b42286b 100644 --- a/tests/exporters/test_export.py +++ b/tests/exporters/test_export.py @@ -234,11 +234,17 @@ # Exported and run as usual, but not compared against eager (see the ``.exactness`` note above). "onnx.exactness": { "Wav2Vec2ForPreTraining": ( - "`codevector_perplexity` and `projected_quantized_states` come out of a Gumbel-softmax draw " - "inside the forward, so the quantizer picks different codes each run and eager does not agree " - "with itself either." + "In eval mode the codevector quantizer picks one of 320 codes per frame with a hard `argmax`. " + "With random weights, two code logits can land within a few float32 ulps of each other, and " + "ORT's rounding (~1e-5 on the logits) then picks the other code. One flipped code changes that " + "frame's `projected_quantized_states` (~0.5% of the elements, within the 5% allowance) and " + "makes the scalar `codevector_perplexity` jump by up to ~1%, far over the tolerance." + ), + "UniSpeechForPreTraining": "Same `argmax` near-tie in the codevector quantizer as `Wav2Vec2ForPreTraining`.", + "UniSpeechSatForPreTraining": "Same `argmax` near-tie in the codevector quantizer as `Wav2Vec2ForPreTraining`.", + "Wav2Vec2ConformerForPreTraining": ( + "Same `argmax` near-tie in the codevector quantizer as `Wav2Vec2ForPreTraining`." ), - "UniSpeechForPreTraining": "Same Gumbel-softmax quantizer draw as `Wav2Vec2ForPreTraining`.", "PatchTSTForPretraining": ( "The tiny config sets `random_mask_ratio=0.0`, so nothing is masked and the pretraining loss is " "`0 / (0 + 1e-10)`: eager reports a clean `0.0` while ORT lands on `NaN`. The reconstruction it " From 60cbcdd3a02c9c4e114f4362fc3daf62eff1598a Mon Sep 17 00:00:00 2001 From: ydshieh Date: Tue, 10 Nov 2026 15:27:53 +0100 Subject: [PATCH 2/2] Shorten the `onnx.exactness` reasons, refer to the PR for details Co-Authored-By: Claude Opus 5.5 --- tests/exporters/test_export.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/exporters/test_export.py b/tests/exporters/test_export.py index 30bc6b42286b..f3f839b7b8c8 100644 --- a/tests/exporters/test_export.py +++ b/tests/exporters/test_export.py @@ -234,17 +234,12 @@ # Exported and run as usual, but not compared against eager (see the ``.exactness`` note above). "onnx.exactness": { "Wav2Vec2ForPreTraining": ( - "In eval mode the codevector quantizer picks one of 320 codes per frame with a hard `argmax`. " - "With random weights, two code logits can land within a few float32 ulps of each other, and " - "ORT's rounding (~1e-5 on the logits) then picks the other code. One flipped code changes that " - "frame's `projected_quantized_states` (~0.5% of the elements, within the 5% allowance) and " - "makes the scalar `codevector_perplexity` jump by up to ~1%, far over the tolerance." - ), - "UniSpeechForPreTraining": "Same `argmax` near-tie in the codevector quantizer as `Wav2Vec2ForPreTraining`.", - "UniSpeechSatForPreTraining": "Same `argmax` near-tie in the codevector quantizer as `Wav2Vec2ForPreTraining`.", - "Wav2Vec2ConformerForPreTraining": ( - "Same `argmax` near-tie in the codevector quantizer as `Wav2Vec2ForPreTraining`." + "The quantizer picks codes with `argmax`; on near-tied logits ORT's rounding can pick another " + "code, and `codevector_perplexity` jumps far over the tolerance. See PR #50575." ), + "UniSpeechForPreTraining": "Same `argmax` near-tie as `Wav2Vec2ForPreTraining`.", + "UniSpeechSatForPreTraining": "Same `argmax` near-tie as `Wav2Vec2ForPreTraining`.", + "Wav2Vec2ConformerForPreTraining": "Same `argmax` near-tie as `Wav2Vec2ForPreTraining`.", "PatchTSTForPretraining": ( "The tiny config sets `random_mask_ratio=0.0`, so nothing is masked and the pretraining loss is " "`0 / (0 + 1e-10)`: eager reports a clean `0.0` while ORT lands on `NaN`. The reconstruction it "