This is for bugs only
Did you already ask in the discord?
Yes
You verified that this is a bug and not a feature request or question by asking in the discord?
Yes
Describe the bug
Unable to train MiniMaxH3 LORA (and LTX).
Depending on settings either it crashes (Layered Offloading disabled) or OOM (Layered Offloading enabled).
Similar issue with LTX-2.3. In the past I was able to train LTX-2.3 now same workflow (same settings) errors out similarly to H3.
Observed behaviour: RAM spikes to 120GB+, hits swap file (uses 20GB+), reduces usage to around 80GB+, and fails with cache enabled.
Spec: Windows 10, 96GB RAM, RTX5090, Python 3.12, fallowed manual install on git page and used "AI-Toolkit-Easy-Install" both behave in the same way.
Did sanity check and used Anima, works fine.
Error with Layered Offloading disabled (truncade to error output only):
Caching text embeddings to disk: 0%| | 0/10 [00:00<?, ?it/s]
Error running job: Command '['Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\tcc\tcc.exe', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb\cuda_utils.c', '-O3', '-shared', '-Wno-psabi', '-o', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb\cuda_utils.cp312-win_amd64.pyd', '-fPIC', '-D_Py_USE_GCC_BUILTIN_ATOMICS', '-lcuda', '-lpython312', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib\x64', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-I\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb', '-IY:\AI_ToolKit2\python_embeded\Include']' returned non-zero exit status 1.
========================================
Result:
- 0 completed jobs
- 1 failure
========================================
Traceback (most recent call last):
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 147, in
main()
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 132, in main
raise e
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 120, in main
job.run()
File "Y:\AI_ToolKit2\ai-toolkit\jobs\ExtensionJob.py", line 22, in run
process.run()
File "Y:\AI_ToolKit2\ai-toolkit\jobs\process\BaseSDTrainProcess.py", line 2252, in run
self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 709, in get_dataloader_from_datasets
dataset = AiToolkitDataset(config, batch_size=dataset_batch_size, sd=sd)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 600, in init
self.setup_epoch()
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 614, in setup_epoch
self.cache_text_embeddings()
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\dataloader_mixins.py", line 2402, in cache_text_embeddings
prompt_embeds: PromptEmbeds = self.sd.encode_prompt(caption)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\models\base_model.py", line 1129, in encode_prompt
return self.get_prompt_embeds(prompt, control_images=control_images)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\extensions_built_in\diffusion_models\minimax_h3\minimax_h3.py", line 720, in get_prompt_embeds
embeds, tags = encode_minimax_h3_prompt(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\utils_contextlib.py", line 124, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\extensions_built_in\diffusion_models\minimax_h3\src\text_encoder.py", line 244, in encode_minimax_h3_prompt
outputs = text_encoder.model(
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\generic.py", line 876, in wrapper
output = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 1346, in forward
outputs = self.language_model(
^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\generic.py", line 952, in wrapper
output = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\output_capturing.py", line 248, in wrapper
outputs = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 916, in forward
layer_outputs = decoder_layer(
^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\modeling_layers.py", line 93, in call
return super().call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1884, in _call_impl
return inner()
^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1832, in inner
result = forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 548, in forward
hidden_states, _ = self.self_attn(
^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1884, in _call_impl
return inner()
^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1832, in inner
result = forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 478, in forward
query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\memory_management\manager_modules.py", line 786, in _mm_forward
out = self._original_forward(x)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\ostris_quant.py", line 123, in forward
return self.ostris_quantizer.forward(self, x)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\nvfp4_quant.py", line 154, in forward
w = self._dequantize_weight(module, x.dtype)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\nvfp4_quant.py", line 117, in _dequantize_weight
return dequantize_nvfp4(
^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\convrot_quant.py", line 247, in dequantize_nvfp4
return _fp4_dequant_op(
^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 863, in call
return self._opoverload(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_ops.py", line 875, in call
return self._op(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\autograd.py", line 123, in autograd_impl
result = forward_no_grad(*args, Metadata(keyset, keyword_only_args))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\autograd.py", line 41, in forward_no_grad
result = op.redispatch(keyset & _C._after_autograd_keyset, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_ops.py", line 882, in redispatch
return self._handle.redispatch_boxed(keyset, *args, **kwargs) # type: ignore[return-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 441, in backend_impl
result = self._backend_fns[device_type](*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_compile.py", line 54, in inner
return disable_fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_dynamo\eval_frame.py", line 1446, in _fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 502, in wrapped_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\convrot_quant.py", line 540, in _fp4_dequant_op
kernel[grid](
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\jit.py", line 370, in
return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\jit.py", line 713, in run
device = driver.active.get_current_device()
^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 52, in active
self._active = self.default
^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 46, in default
self._default = _create_driver()
^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 34, in _create_driver
return active_drivers0
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\driver.py", line 347, in init
self.utils = CudaUtils() # TODO: make static
^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\driver.py", line 77, in init
mod = compile_module_from_src(
^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 193, in compile_module_from_src
so = _build(name, src_path, tmpdir, library_dirs or [], include_dirs or [], libraries or [], ccflags or [])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 139, in _build
raise e
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 136, in _build
subprocess.check_call(cc_cmd)
File "subprocess.py", line 413, in check_call
subprocess.CalledProcessError: Command '['Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\tcc\tcc.exe', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb\cuda_utils.c', '-O3', '-shared', '-Wno-psabi', '-o', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb\cuda_utils.cp312-win_amd64.pyd', '-fPIC', '-D_Py_USE_GCC_BUILTIN_ATOMICS', '-lcuda', '-lpython312', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib\x64', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-I\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb', '-IY:\AI_ToolKit2\python_embeded\Include']' returned non-zero exit status 1.
Error with Layered Offloading enabled (log, contains settings also):
Running 1 job
{
"type": "diffusion_trainer",
"training_folder": "Y:\AI_ToolKit2\ai-toolkit\output",
"sqlite_db_path": "Y:\AI_ToolKit2\ai-toolkit\aitk_db.db",
"device": "cuda",
"trigger_word": null,
"performance_log_every": 10,
"network": {
"type": "lora",
"linear": 16,
"linear_alpha": 16,
"conv": 16,
"conv_alpha": 16,
"lokr_full_rank": true,
"lokr_factor": -1,
"network_kwargs": {
"ignore_if_contains": [
"adaln_proj"
]
}
},
"save": {
"dtype": "bf16",
"save_every": 10,
"max_step_saves_to_keep": 12,
"save_format": "diffusers",
"push_to_hub": false
},
"datasets": [
{
"folder_path": "Y:\AI_ToolKit2\ai-toolkit\datasets/amaz",
"mask_path": null,
"mask_min_value": 0.1,
"default_caption": "",
"caption_ext": "txt",
"caption_dropout_rate": 0.02,
"cache_latents_to_disk": true,
"is_reg": false,
"network_weight": 1,
"resolution": [
256
],
"controls": [],
"shrink_video_to_frames": true,
"flip_x": false,
"flip_y": false,
"num_repeats": 1,
"do_i2v": false,
"fps": 24,
"num_frames": 1,
"auto_frame_count": false
}
],
"train": {
"batch_size": 1,
"bypass_guidance_embedding": false,
"steps": 150,
"gradient_accumulation": 1,
"train_unet": true,
"train_text_encoder": false,
"gradient_checkpointing": true,
"noise_scheduler": "flowmatch",
"optimizer": "adamw8bit",
"timestep_type": "shift",
"content_or_style": "balanced",
"optimizer_params": {
"weight_decay": 0.0001
},
"unload_text_encoder": false,
"cache_text_embeddings": true,
"lr": 0.0001,
"ema_config": {
"use_ema": false,
"ema_decay": 0.99
},
"skip_first_sample": false,
"force_first_sample": false,
"disable_sampling": false,
"dtype": "bf16",
"diff_output_preservation": false,
"diff_output_preservation_multiplier": 1,
"diff_output_preservation_class": "person",
"switch_boundary_every": 1,
"loss_type": "mse",
"do_guidance_loss": true,
"guidance_loss_target": 3.5,
"audio_loss_multiplier": 1
},
"logging": {
"log_every": 1,
"use_ui_logger": true
},
"model": {
"name_or_path": "Comfy-Org/MiniMax-H3",
"quantize": true,
"qtype": "convrot8",
"quantize_te": true,
"qtype_te": "nvfp4",
"arch": "minimax_h3",
"low_vram": true,
"model_kwargs": {},
"compile": false,
"layer_offloading": true,
"layer_offloading_text_encoder_percent": 1,
"layer_offloading_transformer_percent": 1
},
"sample": {
"sampler": "flowmatch",
"sample_every": 10,
"sample_start_step": 0,
"width": 768,
"height": 768,
"samples": [
{
"prompt": "Amazon, 1girl, solo, long hair, looking at viewer, simple background, blonde hair, white background, closed mouth, standing, full body, ponytail, armor, thigh strap, sandals, high ponytail"
}
],
"neg": "",
"seed": 42,
"walk_seed": true,
"guidance_scale": 1,
"sample_steps": 28,
"num_frames": 107,
"fps": 24
}
}
Changing sample extention to animated webp
Using SQLite database at Y:\AI_ToolKit2\ai-toolkit\aitk_db.db
Job ID: "fbb2602c-a17b-4fe1-a483-f127186d0b77"
#############################################
Running job: hhm3_amaz_2
#############################################
Running 1 process
Loading MiniMax-H3 model
Loading transformer from Y:\AI_ToolKit2\ai-toolkit\models\diffusion_models/minimax_h3_fl2va_pruned_int8_convrot.safetensors
- attached 200 pre-quantized ConvRot layers
Quantizing transformer
- quantizing 50 transformer blocks
0%| | 0/50 [00:00<?, ?it/s]
100%|##########| 50/50 [00:00<?, ?it/s]
- 50 blocks already quantized with a matching qtype; left untouched
- quantizing extras
Keeping transformer on CPU
Loading Qwen3-VL text encoder from Y:\AI_ToolKit2\ai-toolkit\models\text_encoders/qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors
- attached 351 pre-quantized nvfp4/int8 layers
Text encoder is already nvfp4/int8 quantized; skipping quantize_te
Loading video VAE
Loading audio VAE
Model Loaded
create LoRA network. base dim (rank): 16, alpha: 16
neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
create LoRA for Text Encoder: 0 modules.
create LoRA for U-Net: 208 modules.
enable LoRA for U-Net
Dataset: Y:\AI_ToolKit2\ai-toolkit\datasets/amaz
- Preprocessing image dimensions
0%| | 0/10 [00:00<?, ?it/s]
100%|##########| 10/10 [00:00<?, ?it/s]
- Found 10 images
Bucket sizes for Y:\AI_ToolKit2\ai-toolkit\datasets/amaz:
224x288: 2 files
128x448: 2 files
128x512: 1 files
256x256: 3 files
128x480: 1 files
192x320: 1 files
6 buckets made
Caching latents for Y:\AI_ToolKit2\ai-toolkit\datasets/amaz
- Saving latents to disk
Caching latents to disk: 0%| | 0/10 [00:00<?, ?it/s]
Caching latents to disk: 10%|# | 1/10 [00:00<00:01, 5.44it/s]
Caching latents to disk: 100%|##########| 10/10 [00:00<00:00, 54.40it/s]
Caching text_embeddings for Y:\AI_ToolKit2\ai-toolkit\datasets/amaz
- Saving text embeddings to disk
Caching text embeddings to disk: 0%| | 0/10 [00:00<?, ?it/s]Failed to compile. cc_cmd: ['Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\tcc\tcc.exe', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.c', '-O3', '-shared', '-Wno-psabi', '-o', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.cp312-win_amd64.pyd', '-fPIC', '-D_Py_USE_GCC_BUILTIN_ATOMICS', '-lcuda', '-lpython312', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib\x64', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-I\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow', '-IY:\AI_ToolKit2\python_embeded\Include']
Caching text embeddings to disk: 0%| | 0/10 [00:00<?, ?it/s]
Error running job: Command '['Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\tcc\tcc.exe', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.c', '-O3', '-shared', '-Wno-psabi', '-o', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.cp312-win_amd64.pyd', '-fPIC', '-D_Py_USE_GCC_BUILTIN_ATOMICS', '-lcuda', '-lpython312', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib\x64', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-I\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow', '-IY:\AI_ToolKit2\python_embeded\Include']' returned non-zero exit status 1.
========================================
Result:
- 0 completed jobs
- 1 failure
========================================
Traceback (most recent call last):
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 147, in
main()
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 132, in main
raise e
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 120, in main
job.run()
File "Y:\AI_ToolKit2\ai-toolkit\jobs\ExtensionJob.py", line 22, in run
process.run()
File "Y:\AI_ToolKit2\ai-toolkit\jobs\process\BaseSDTrainProcess.py", line 2252, in run
self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 709, in get_dataloader_from_datasets
dataset = AiToolkitDataset(config, batch_size=dataset_batch_size, sd=sd)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 600, in init
self.setup_epoch()
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 614, in setup_epoch
self.cache_text_embeddings()
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\dataloader_mixins.py", line 2402, in cache_text_embeddings
prompt_embeds: PromptEmbeds = self.sd.encode_prompt(caption)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\models\base_model.py", line 1129, in encode_prompt
return self.get_prompt_embeds(prompt, control_images=control_images)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\extensions_built_in\diffusion_models\minimax_h3\minimax_h3.py", line 720, in get_prompt_embeds
embeds, tags = encode_minimax_h3_prompt(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\utils_contextlib.py", line 124, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\extensions_built_in\diffusion_models\minimax_h3\src\text_encoder.py", line 244, in encode_minimax_h3_prompt
outputs = text_encoder.model(
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\generic.py", line 876, in wrapper
output = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 1346, in forward
outputs = self.language_model(
^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\generic.py", line 952, in wrapper
output = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\output_capturing.py", line 248, in wrapper
outputs = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 916, in forward
layer_outputs = decoder_layer(
^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\modeling_layers.py", line 93, in call
return super().call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1884, in _call_impl
return inner()
^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1832, in inner
result = forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 548, in forward
hidden_states, _ = self.self_attn(
^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1884, in _call_impl
return inner()
^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1832, in inner
result = forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 478, in forward
query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\memory_management\manager_modules.py", line 786, in _mm_forward
out = self._original_forward(x)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\ostris_quant.py", line 123, in forward
return self.ostris_quantizer.forward(self, x)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\nvfp4_quant.py", line 154, in forward
w = self._dequantize_weight(module, x.dtype)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\nvfp4_quant.py", line 117, in _dequantize_weight
return dequantize_nvfp4(
^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\convrot_quant.py", line 247, in dequantize_nvfp4
return _fp4_dequant_op(
^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 863, in call
return self._opoverload(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_ops.py", line 875, in call
return self._op(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\autograd.py", line 123, in autograd_impl
result = forward_no_grad(*args, Metadata(keyset, keyword_only_args))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\autograd.py", line 41, in forward_no_grad
result = op.redispatch(keyset & _C._after_autograd_keyset, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_ops.py", line 882, in redispatch
return self._handle.redispatch_boxed(keyset, *args, **kwargs) # type: ignore[return-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 441, in backend_impl
result = self._backend_fns[device_type](*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_compile.py", line 54, in inner
return disable_fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_dynamo\eval_frame.py", line 1446, in _fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 502, in wrapped_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\convrot_quant.py", line 540, in _fp4_dequant_op
kernel[grid](
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\jit.py", line 370, in
return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\jit.py", line 713, in run
device = driver.active.get_current_device()
^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 52, in active
self._active = self.default
^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 46, in default
self._default = _create_driver()
^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 34, in _create_driver
return active_drivers0
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\driver.py", line 347, in init
self.utils = CudaUtils() # TODO: make static
^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\driver.py", line 77, in init
mod = compile_module_from_src(
^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 193, in compile_module_from_src
so = _build(name, src_path, tmpdir, library_dirs or [], include_dirs or [], libraries or [], ccflags or [])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 139, in _build
raise e
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 136, in _build
subprocess.check_call(cc_cmd)
File "subprocess.py", line 413, in check_call
subprocess.CalledProcessError: Command '['Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\tcc\tcc.exe', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.c', '-O3', '-shared', '-Wno-psabi', '-o', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.cp312-win_amd64.pyd', '-fPIC', '-D_Py_USE_GCC_BUILTIN_ATOMICS', '-lcuda', '-lpython312', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib\x64', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-I\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow', '-IY:\AI_ToolKit2\python_embeded\Include']' returned non-zero exit status 1.
This is for bugs only
Did you already ask in the discord?
Yes
You verified that this is a bug and not a feature request or question by asking in the discord?
Yes
Describe the bug
Unable to train MiniMaxH3 LORA (and LTX).
Depending on settings either it crashes (Layered Offloading disabled) or OOM (Layered Offloading enabled).
Similar issue with LTX-2.3. In the past I was able to train LTX-2.3 now same workflow (same settings) errors out similarly to H3.
Observed behaviour: RAM spikes to 120GB+, hits swap file (uses 20GB+), reduces usage to around 80GB+, and fails with cache enabled.
Spec: Windows 10, 96GB RAM, RTX5090, Python 3.12, fallowed manual install on git page and used "AI-Toolkit-Easy-Install" both behave in the same way.
Did sanity check and used Anima, works fine.
Error with Layered Offloading disabled (truncade to error output only):
Caching text embeddings to disk: 0%| | 0/10 [00:00<?, ?it/s]
Error running job: Command '['Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\tcc\tcc.exe', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb\cuda_utils.c', '-O3', '-shared', '-Wno-psabi', '-o', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb\cuda_utils.cp312-win_amd64.pyd', '-fPIC', '-D_Py_USE_GCC_BUILTIN_ATOMICS', '-lcuda', '-lpython312', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib\x64', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-I\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb', '-IY:\AI_ToolKit2\python_embeded\Include']' returned non-zero exit status 1.
========================================
Result:
========================================
Traceback (most recent call last):
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 147, in
main()
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 132, in main
raise e
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 120, in main
job.run()
File "Y:\AI_ToolKit2\ai-toolkit\jobs\ExtensionJob.py", line 22, in run
process.run()
File "Y:\AI_ToolKit2\ai-toolkit\jobs\process\BaseSDTrainProcess.py", line 2252, in run
self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 709, in get_dataloader_from_datasets
dataset = AiToolkitDataset(config, batch_size=dataset_batch_size, sd=sd)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 600, in init
self.setup_epoch()
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 614, in setup_epoch
self.cache_text_embeddings()
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\dataloader_mixins.py", line 2402, in cache_text_embeddings
prompt_embeds: PromptEmbeds = self.sd.encode_prompt(caption)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\models\base_model.py", line 1129, in encode_prompt
return self.get_prompt_embeds(prompt, control_images=control_images)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\extensions_built_in\diffusion_models\minimax_h3\minimax_h3.py", line 720, in get_prompt_embeds
embeds, tags = encode_minimax_h3_prompt(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\utils_contextlib.py", line 124, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\extensions_built_in\diffusion_models\minimax_h3\src\text_encoder.py", line 244, in encode_minimax_h3_prompt
outputs = text_encoder.model(
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\generic.py", line 876, in wrapper
output = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 1346, in forward
outputs = self.language_model(
^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\generic.py", line 952, in wrapper
output = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\output_capturing.py", line 248, in wrapper
outputs = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 916, in forward
layer_outputs = decoder_layer(
^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\modeling_layers.py", line 93, in call
return super().call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1884, in _call_impl
return inner()
^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1832, in inner
result = forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 548, in forward
hidden_states, _ = self.self_attn(
^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1884, in _call_impl
return inner()
^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1832, in inner
result = forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 478, in forward
query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\memory_management\manager_modules.py", line 786, in _mm_forward
out = self._original_forward(x)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\ostris_quant.py", line 123, in forward
return self.ostris_quantizer.forward(self, x)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\nvfp4_quant.py", line 154, in forward
w = self._dequantize_weight(module, x.dtype)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\nvfp4_quant.py", line 117, in _dequantize_weight
return dequantize_nvfp4(
^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\convrot_quant.py", line 247, in dequantize_nvfp4
return _fp4_dequant_op(
^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 863, in call
return self._opoverload(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_ops.py", line 875, in call
return self._op(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\autograd.py", line 123, in autograd_impl
result = forward_no_grad(*args, Metadata(keyset, keyword_only_args))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\autograd.py", line 41, in forward_no_grad
result = op.redispatch(keyset & _C._after_autograd_keyset, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_ops.py", line 882, in redispatch
return self._handle.redispatch_boxed(keyset, *args, **kwargs) # type: ignore[return-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 441, in backend_impl
result = self._backend_fns[device_type](*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_compile.py", line 54, in inner
return disable_fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_dynamo\eval_frame.py", line 1446, in _fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 502, in wrapped_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\convrot_quant.py", line 540, in _fp4_dequant_op
kernel[grid](
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\jit.py", line 370, in
return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\jit.py", line 713, in run
device = driver.active.get_current_device()
^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 52, in active
self._active = self.default
^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 46, in default
self._default = _create_driver()
^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 34, in _create_driver
return active_drivers0
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\driver.py", line 347, in init
self.utils = CudaUtils() # TODO: make static
^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\driver.py", line 77, in init
mod = compile_module_from_src(
^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 193, in compile_module_from_src
so = _build(name, src_path, tmpdir, library_dirs or [], include_dirs or [], libraries or [], ccflags or [])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 139, in _build
raise e
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 136, in _build
subprocess.check_call(cc_cmd)
File "subprocess.py", line 413, in check_call
subprocess.CalledProcessError: Command '['Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\tcc\tcc.exe', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb\cuda_utils.c', '-O3', '-shared', '-Wno-psabi', '-o', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb\cuda_utils.cp312-win_amd64.pyd', '-fPIC', '-D_Py_USE_GCC_BUILTIN_ATOMICS', '-lcuda', '-lpython312', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib\x64', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-I\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpe9swn5qb', '-IY:\AI_ToolKit2\python_embeded\Include']' returned non-zero exit status 1.
Error with Layered Offloading enabled (log, contains settings also):
Running 1 job
{
"type": "diffusion_trainer",
"training_folder": "Y:\AI_ToolKit2\ai-toolkit\output",
"sqlite_db_path": "Y:\AI_ToolKit2\ai-toolkit\aitk_db.db",
"device": "cuda",
"trigger_word": null,
"performance_log_every": 10,
"network": {
"type": "lora",
"linear": 16,
"linear_alpha": 16,
"conv": 16,
"conv_alpha": 16,
"lokr_full_rank": true,
"lokr_factor": -1,
"network_kwargs": {
"ignore_if_contains": [
"adaln_proj"
]
}
},
"save": {
"dtype": "bf16",
"save_every": 10,
"max_step_saves_to_keep": 12,
"save_format": "diffusers",
"push_to_hub": false
},
"datasets": [
{
"folder_path": "Y:\AI_ToolKit2\ai-toolkit\datasets/amaz",
"mask_path": null,
"mask_min_value": 0.1,
"default_caption": "",
"caption_ext": "txt",
"caption_dropout_rate": 0.02,
"cache_latents_to_disk": true,
"is_reg": false,
"network_weight": 1,
"resolution": [
256
],
"controls": [],
"shrink_video_to_frames": true,
"flip_x": false,
"flip_y": false,
"num_repeats": 1,
"do_i2v": false,
"fps": 24,
"num_frames": 1,
"auto_frame_count": false
}
],
"train": {
"batch_size": 1,
"bypass_guidance_embedding": false,
"steps": 150,
"gradient_accumulation": 1,
"train_unet": true,
"train_text_encoder": false,
"gradient_checkpointing": true,
"noise_scheduler": "flowmatch",
"optimizer": "adamw8bit",
"timestep_type": "shift",
"content_or_style": "balanced",
"optimizer_params": {
"weight_decay": 0.0001
},
"unload_text_encoder": false,
"cache_text_embeddings": true,
"lr": 0.0001,
"ema_config": {
"use_ema": false,
"ema_decay": 0.99
},
"skip_first_sample": false,
"force_first_sample": false,
"disable_sampling": false,
"dtype": "bf16",
"diff_output_preservation": false,
"diff_output_preservation_multiplier": 1,
"diff_output_preservation_class": "person",
"switch_boundary_every": 1,
"loss_type": "mse",
"do_guidance_loss": true,
"guidance_loss_target": 3.5,
"audio_loss_multiplier": 1
},
"logging": {
"log_every": 1,
"use_ui_logger": true
},
"model": {
"name_or_path": "Comfy-Org/MiniMax-H3",
"quantize": true,
"qtype": "convrot8",
"quantize_te": true,
"qtype_te": "nvfp4",
"arch": "minimax_h3",
"low_vram": true,
"model_kwargs": {},
"compile": false,
"layer_offloading": true,
"layer_offloading_text_encoder_percent": 1,
"layer_offloading_transformer_percent": 1
},
"sample": {
"sampler": "flowmatch",
"sample_every": 10,
"sample_start_step": 0,
"width": 768,
"height": 768,
"samples": [
{
"prompt": "Amazon, 1girl, solo, long hair, looking at viewer, simple background, blonde hair, white background, closed mouth, standing, full body, ponytail, armor, thigh strap, sandals, high ponytail"
}
],
"neg": "",
"seed": 42,
"walk_seed": true,
"guidance_scale": 1,
"sample_steps": 28,
"num_frames": 107,
"fps": 24
}
}
Changing sample extention to animated webp
Using SQLite database at Y:\AI_ToolKit2\ai-toolkit\aitk_db.db
Job ID: "fbb2602c-a17b-4fe1-a483-f127186d0b77"
#############################################
Running job: hhm3_amaz_2
#############################################
Running 1 process
Loading MiniMax-H3 model
Loading transformer from Y:\AI_ToolKit2\ai-toolkit\models\diffusion_models/minimax_h3_fl2va_pruned_int8_convrot.safetensors
Quantizing transformer
0%| | 0/50 [00:00<?, ?it/s]
100%|##########| 50/50 [00:00<?, ?it/s]
Keeping transformer on CPU
Loading Qwen3-VL text encoder from Y:\AI_ToolKit2\ai-toolkit\models\text_encoders/qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors
Text encoder is already nvfp4/int8 quantized; skipping quantize_te
Loading video VAE
Loading audio VAE
Model Loaded
create LoRA network. base dim (rank): 16, alpha: 16
neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
create LoRA for Text Encoder: 0 modules.
create LoRA for U-Net: 208 modules.
enable LoRA for U-Net
Dataset: Y:\AI_ToolKit2\ai-toolkit\datasets/amaz
0%| | 0/10 [00:00<?, ?it/s]
100%|##########| 10/10 [00:00<?, ?it/s]
Bucket sizes for Y:\AI_ToolKit2\ai-toolkit\datasets/amaz:
224x288: 2 files
128x448: 2 files
128x512: 1 files
256x256: 3 files
128x480: 1 files
192x320: 1 files
6 buckets made
Caching latents for Y:\AI_ToolKit2\ai-toolkit\datasets/amaz
Caching latents to disk: 0%| | 0/10 [00:00<?, ?it/s]
Caching latents to disk: 10%|# | 1/10 [00:00<00:01, 5.44it/s]
Caching latents to disk: 100%|##########| 10/10 [00:00<00:00, 54.40it/s]
Caching text_embeddings for Y:\AI_ToolKit2\ai-toolkit\datasets/amaz
Caching text embeddings to disk: 0%| | 0/10 [00:00<?, ?it/s]Failed to compile. cc_cmd: ['Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\tcc\tcc.exe', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.c', '-O3', '-shared', '-Wno-psabi', '-o', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.cp312-win_amd64.pyd', '-fPIC', '-D_Py_USE_GCC_BUILTIN_ATOMICS', '-lcuda', '-lpython312', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib\x64', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-I\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow', '-IY:\AI_ToolKit2\python_embeded\Include']
Caching text embeddings to disk: 0%| | 0/10 [00:00<?, ?it/s]
Error running job: Command '['Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\tcc\tcc.exe', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.c', '-O3', '-shared', '-Wno-psabi', '-o', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.cp312-win_amd64.pyd', '-fPIC', '-D_Py_USE_GCC_BUILTIN_ATOMICS', '-lcuda', '-lpython312', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib\x64', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-I\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow', '-IY:\AI_ToolKit2\python_embeded\Include']' returned non-zero exit status 1.
========================================
Result:
========================================
Traceback (most recent call last):
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 147, in
main()
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 132, in main
raise e
File "Y:\AI_ToolKit2\ai-toolkit\run.py", line 120, in main
job.run()
File "Y:\AI_ToolKit2\ai-toolkit\jobs\ExtensionJob.py", line 22, in run
process.run()
File "Y:\AI_ToolKit2\ai-toolkit\jobs\process\BaseSDTrainProcess.py", line 2252, in run
self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 709, in get_dataloader_from_datasets
dataset = AiToolkitDataset(config, batch_size=dataset_batch_size, sd=sd)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 600, in init
self.setup_epoch()
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\data_loader.py", line 614, in setup_epoch
self.cache_text_embeddings()
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\dataloader_mixins.py", line 2402, in cache_text_embeddings
prompt_embeds: PromptEmbeds = self.sd.encode_prompt(caption)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\models\base_model.py", line 1129, in encode_prompt
return self.get_prompt_embeds(prompt, control_images=control_images)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\extensions_built_in\diffusion_models\minimax_h3\minimax_h3.py", line 720, in get_prompt_embeds
embeds, tags = encode_minimax_h3_prompt(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\utils_contextlib.py", line 124, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\extensions_built_in\diffusion_models\minimax_h3\src\text_encoder.py", line 244, in encode_minimax_h3_prompt
outputs = text_encoder.model(
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\generic.py", line 876, in wrapper
output = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 1346, in forward
outputs = self.language_model(
^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\generic.py", line 952, in wrapper
output = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\utils\output_capturing.py", line 248, in wrapper
outputs = func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 916, in forward
layer_outputs = decoder_layer(
^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\modeling_layers.py", line 93, in call
return super().call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1884, in _call_impl
return inner()
^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1832, in inner
result = forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 548, in forward
hidden_states, _ = self.self_attn(
^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1884, in _call_impl
return inner()
^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1832, in inner
result = forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\transformers\models\qwen3_vl\modeling_qwen3_vl.py", line 478, in forward
query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1778, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch\nn\modules\module.py", line 1789, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\memory_management\manager_modules.py", line 786, in _mm_forward
out = self._original_forward(x)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\ostris_quant.py", line 123, in forward
return self.ostris_quantizer.forward(self, x)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\nvfp4_quant.py", line 154, in forward
w = self._dequantize_weight(module, x.dtype)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\nvfp4_quant.py", line 117, in _dequantize_weight
return dequantize_nvfp4(
^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\convrot_quant.py", line 247, in dequantize_nvfp4
return _fp4_dequant_op(
^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 863, in call
return self._opoverload(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_ops.py", line 875, in call
return self._op(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\autograd.py", line 123, in autograd_impl
result = forward_no_grad(*args, Metadata(keyset, keyword_only_args))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\autograd.py", line 41, in forward_no_grad
result = op.redispatch(keyset & _C._after_autograd_keyset, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_ops.py", line 882, in redispatch
return self._handle.redispatch_boxed(keyset, *args, **kwargs) # type: ignore[return-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 441, in backend_impl
result = self._backend_fns[device_type](*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_compile.py", line 54, in inner
return disable_fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_dynamo\eval_frame.py", line 1446, in _fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\torch_library\custom_ops.py", line 502, in wrapped_fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\ai-toolkit\toolkit\util\convrot_quant.py", line 540, in _fp4_dequant_op
kernel[grid](
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\jit.py", line 370, in
return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\jit.py", line 713, in run
device = driver.active.get_current_device()
^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 52, in active
self._active = self.default
^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 46, in default
self._default = _create_driver()
^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\driver.py", line 34, in _create_driver
return active_drivers0
^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\driver.py", line 347, in init
self.utils = CudaUtils() # TODO: make static
^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\driver.py", line 77, in init
mod = compile_module_from_src(
^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 193, in compile_module_from_src
so = _build(name, src_path, tmpdir, library_dirs or [], include_dirs or [], libraries or [], ccflags or [])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 139, in _build
raise e
File "Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\build.py", line 136, in _build
subprocess.check_call(cc_cmd)
File "subprocess.py", line 413, in check_call
subprocess.CalledProcessError: Command '['Y:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\runtime\tcc\tcc.exe', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.c', '-O3', '-shared', '-Wno-psabi', '-o', '\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow\cuda_utils.cp312-win_amd64.pyd', '-fPIC', '-D_Py_USE_GCC_BUILTIN_ATOMICS', '-lcuda', '-lpython312', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib', '-LY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\lib\x64', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-IY:\AI_ToolKit2\python_embeded\Lib\site-packages\triton\backends\nvidia\include', '-I\\?\Y:\AI_ToolKit2\User\AppData\Local\Temp\tmpzygvmxow', '-IY:\AI_ToolKit2\python_embeded\Include']' returned non-zero exit status 1.