diff --git a/__init__.py b/__init__.py index 11674484..2d6cb27e 100644 --- a/__init__.py +++ b/__init__.py @@ -49,6 +49,13 @@ except ImportError: logger.exception("Nodes `NunchakuFluxLoraLoader` and `NunchakuFluxLoraStack` import failed:") +try: + from .nodes.lora.qwenimage import NunchakuQwenImageLoraLoader + + NODE_CLASS_MAPPINGS["NunchakuQwenImageLoraLoader"] = NunchakuQwenImageLoraLoader +except ImportError: + logger.exception("Node `NunchakuQwenImageLoraLoader` import failed:") + try: from .nodes.models.text_encoder import NunchakuTextEncoderLoader, NunchakuTextEncoderLoaderV2 diff --git a/example_workflows/nunchaku-qwen-image-edit-lora.json b/example_workflows/nunchaku-qwen-image-edit-lora.json new file mode 100644 index 00000000..75f8ca1d --- /dev/null +++ b/example_workflows/nunchaku-qwen-image-edit-lora.json @@ -0,0 +1,1071 @@ +{ + "id": "91f6bbe2-ed41-4fd6-bac7-71d5b5864ecb", + "revision": 0, + "last_node_id": 102, + "last_link_id": 190, + "nodes": [ + { + "id": 39, + "type": "VAELoader", + "pos": [ + -250, + 330 + ], + "size": [ + 330, + 60 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "slot_index": 0, + "links": [ + 76, + 161, + 162, + 168 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "VAELoader", + "models": [ + { + "name": "qwen_image_vae.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/vae/qwen_image_vae.safetensors", + "directory": "vae" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": {} + } + }, + "widgets_values": [ + "qwen_image_vae.safetensors" + ] + }, + { + "id": 77, + "type": "TextEncodeQwenImageEdit", + "pos": [ + 140, + 400 + ], + "size": [ + 360, + 150 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 132 + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": 161 + }, + { + "name": "image", + "shape": 7, + "type": "IMAGE", + "link": 180 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 163 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "TextEncodeQwenImageEdit", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": { + "prompt": true + } + } + }, + "widgets_values": [ + "" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 75, + "type": "CFGNorm", + "pos": [ + 550, + 130 + ], + "size": [ + 290, + 60 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 141 + } + ], + "outputs": [ + { + "name": "patched_model", + "type": "MODEL", + "links": [ + 186 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "CFGNorm", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": { + "strength": true + } + } + }, + "widgets_values": [ + 1 + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 890, + 20 + ], + "size": [ + 210, + 46 + ], + "flags": { + "collapsed": false + }, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 128 + }, + { + "name": "vae", + "type": "VAE", + "link": 76 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 110 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "VAEDecode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 88, + "type": "VAEEncode", + "pos": [ + 370, + 630 + ], + "size": [ + 171.66796875, + 46 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "pixels", + "type": "IMAGE", + "link": 178 + }, + { + "name": "vae", + "type": "VAE", + "link": 168 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 170 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "VAEEncode", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": {} + } + }, + "widgets_values": [] + }, + { + "id": 60, + "type": "SaveImage", + "pos": [ + 890, + 240 + ], + "size": [ + 580, + 650 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 110 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "SaveImage", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": {} + } + }, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 76, + "type": "TextEncodeQwenImageEdit", + "pos": [ + 140, + 200 + ], + "size": [ + 360, + 150 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 131 + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": 162 + }, + { + "name": "image", + "shape": 7, + "type": "IMAGE", + "link": 179 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 164 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "TextEncodeQwenImageEdit", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": { + "prompt": true + } + } + }, + "widgets_values": [ + "Remove all UI text elements from the image. Keep the feeling that the characters and scene are in water. Also, remove the green UI elements at the bottom." + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 93, + "type": "ImageScaleToTotalPixels", + "pos": [ + -210, + 890 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 177 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 178, + 179, + 180 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "ImageScaleToTotalPixels", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": { + "upscale_method": true, + "megapixels": true + } + } + }, + "widgets_values": [ + "lanczos", + 1 + ] + }, + { + "id": 99, + "type": "MarkdownNote", + "pos": [ + -825.0533447265625, + -4.325097560882568 + ], + "size": [ + 527.6619873046875, + 185.5856475830078 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Model links", + "properties": { + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": {} + } + }, + "widgets_values": [ + "Download the models from [Hugging Face](https://huggingface.co/nunchaku-tech/nunchaku-qwen-image-edit) or [ModelScope](https://modelscope.cn/models/nunchaku-tech/nunchaku-qwen-image-edit), and place them in the `models/diffusion_models` directory.\n\n- **50-series GPUs:** use the **FP4** models \n- **Other GPUs:** use the **INT4** models \n\nNote: \n- `r128` models provide higher quality than `r32`, but run slightly slower. \n- LoRA support is not available now but will come soon.\n- You can also use the fused lightning models.\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 97, + "type": "MarkdownNote", + "pos": [ + 558.4642944335938, + 772.7610473632812 + ], + "size": [ + 289.96588134765625, + 150.22474670410156 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "KSampler settings", + "properties": { + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": {} + } + }, + "widgets_values": [ + "You can test and find the best setting by yourself. The following table is for reference.\n\n| Model | Steps | CFG |\n|---------------------|---------------|---------------|\n| original | 20 | 2.5 |\n| lightning | 4/8 | 1.0 |\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 96, + "type": "MarkdownNote", + "pos": [ + -550.6171264648438, + 881.757568359375 + ], + "size": [ + 280, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": {} + } + }, + "widgets_values": [ + "This node is to avoid poor output results caused by excessively large input image sizes." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 38, + "type": "CLIPLoader", + "pos": [ + -250, + 170 + ], + "size": [ + 330, + 110 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "slot_index": 0, + "links": [ + 131, + 132 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "CLIPLoader", + "models": [ + { + "name": "qwen_2.5_vl_7b_fp8_scaled.safetensors", + "url": "https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/text_encoders/qwen_2.5_vl_7b_fp8_scaled.safetensors", + "directory": "text_encoders" + } + ], + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": {} + } + }, + "widgets_values": [ + "split_files/text_encoders/qwen_2.5_vl_7b_fp8_scaled.safetensors", + "qwen_image", + "default" + ] + }, + { + "id": 78, + "type": "LoadImage", + "pos": [ + -220, + 500 + ], + "size": [ + 274.080078125, + 314.0000305175781 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 177 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.50", + "Node name for S&R": "LoadImage", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": { + "image": true, + "upload": true + } + } + }, + "widgets_values": [ + "kdb.png", + "image" + ] + }, + { + "id": 101, + "type": "NunchakuQwenImageDiTLoader", + "pos": [ + -242.1013946533203, + 35.88456344604492 + ], + "size": [ + 320.1197204589844, + 130 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 189 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-nunchaku", + "ver": "e11a8016cda7ce1703e1a0a4b5c570988d2825fe", + "Node name for S&R": "NunchakuQwenImageDiTLoader", + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": { + "model_name": true, + "cpu_offload": true, + "num_blocks_on_gpu": true, + "use_pin_memory": true + } + } + }, + "widgets_values": [ + "svdq-int4_r128-qwen-image-edit.safetensors", + "disable", + 1, + "disable" + ] + }, + { + "id": 3, + "type": "KSampler", + "pos": [ + 550, + 240 + ], + "size": [ + 300, + 474 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 186 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 164 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 163 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 170 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "slot_index": 0, + "links": [ + 128 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "KSampler", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": {} + } + }, + "widgets_values": [ + 611075713154559, + "randomize", + 4, + 1, + "euler", + "simple", + 1 + ] + }, + { + "id": 66, + "type": "ModelSamplingAuraFlow", + "pos": [ + 550, + 20 + ], + "size": [ + 290, + 60 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 190 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 141 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48", + "Node name for S&R": "ModelSamplingAuraFlow", + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "ue_properties": { + "version": "7.0.1", + "widget_ue_connectable": {} + } + }, + "widgets_values": [ + 3 + ] + }, + { + "id": 102, + "type": "NunchakuQwenImageLoraLoader", + "pos": [ + 165.70472717285156, + -51.339271545410156 + ], + "size": [ + 333.3677673339844, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 189 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 190 + ] + } + ], + "properties": { + "cnr_id": "ComfyUI-nunchaku", + "ver": "ba3d5de9adecb5a92c70755fb11bb648b6b5a257", + "Node name for S&R": "NunchakuQwenImageLoraLoader", + "ue_properties": { + "version": "7.0.1" + } + }, + "widgets_values": [ + "Qwen-Image-Edit-Lightning-4steps-V1.0-bf16.safetensors", + 1 + ] + } + ], + "links": [ + [ + 76, + 39, + 0, + 8, + 1, + "VAE" + ], + [ + 110, + 8, + 0, + 60, + 0, + "IMAGE" + ], + [ + 128, + 3, + 0, + 8, + 0, + "LATENT" + ], + [ + 131, + 38, + 0, + 76, + 0, + "CLIP" + ], + [ + 132, + 38, + 0, + 77, + 0, + "CLIP" + ], + [ + 141, + 66, + 0, + 75, + 0, + "MODEL" + ], + [ + 161, + 39, + 0, + 77, + 1, + "VAE" + ], + [ + 162, + 39, + 0, + 76, + 1, + "VAE" + ], + [ + 163, + 77, + 0, + 3, + 2, + "CONDITIONING" + ], + [ + 164, + 76, + 0, + 3, + 1, + "CONDITIONING" + ], + [ + 168, + 39, + 0, + 88, + 1, + "VAE" + ], + [ + 170, + 88, + 0, + 3, + 3, + "LATENT" + ], + [ + 177, + 78, + 0, + 93, + 0, + "IMAGE" + ], + [ + 178, + 93, + 0, + 88, + 0, + "IMAGE" + ], + [ + 179, + 93, + 0, + 76, + 2, + "IMAGE" + ], + [ + 180, + 93, + 0, + 77, + 2, + "IMAGE" + ], + [ + 186, + 75, + 0, + 3, + 0, + "MODEL" + ], + [ + 189, + 101, + 0, + 102, + 0, + "MODEL" + ], + [ + 190, + 102, + 0, + 66, + 0, + "MODEL" + ] + ], + "groups": [ + { + "id": 1, + "title": "Step1 - Load models", + "bounding": [ + -270, + -40, + 370, + 450 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Step 2 - Upload image for editing", + "bounding": [ + -270, + 430, + 370, + 400 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Step 3 - Prompt", + "bounding": [ + 130, + 130, + 380, + 433.6000061035156 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7724374270942966, + "offset": [ + 1126.3093117829626, + 314.3678582811355 + ] + }, + "frontendVersion": "1.21.7", + "ue_links": [], + "links_added_by_ue": [], + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} diff --git a/nodes/lora/qwenimage.py b/nodes/lora/qwenimage.py new file mode 100644 index 00000000..dfe43c9a --- /dev/null +++ b/nodes/lora/qwenimage.py @@ -0,0 +1,115 @@ +""" +This module provides the :class:`NunchakuQwenImageLoraLoader` node +for applying LoRA weights to Nunchaku Qwen-Image models within ComfyUI. +""" + +import logging +import os + +import folder_paths + +from nunchaku.lora.flux.v1.lora_flux_v2 import update_lora_params_v2 + +# Get log level from environment variable (default to INFO) +log_level = os.getenv("LOG_LEVEL", "INFO").upper() + +# Configure logging +logging.basicConfig(level=getattr(logging, log_level, logging.INFO), format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger(__name__) + + +class NunchakuQwenImageLoraLoader: + """ + Node for loading and applying a LoRA to a Nunchaku Qwen-Image model. + + Attributes + ---------- + RETURN_TYPES : tuple + The return type of the node ("MODEL",). + OUTPUT_TOOLTIPS : tuple + Tooltip for the output. + FUNCTION : str + The function to call ("load_lora"). + TITLE : str + Node title. + CATEGORY : str + Node category. + DESCRIPTION : str + Node description. + """ + + @classmethod + def INPUT_TYPES(s): + """ + Defines the input types and tooltips for the node. + + Returns + ------- + dict + A dictionary specifying the required inputs and their descriptions for the node interface. + """ + return { + "required": { + "model": ( + "MODEL", + { + "tooltip": "The diffusion model the LoRA will be applied to. " + "Make sure the model is loaded by `Nunchaku Qwen-Image DiT Loader`." + }, + ), + "lora_name": ( + folder_paths.get_filename_list("loras"), + {"tooltip": "The file name of the LoRA."}, + ), + "lora_strength": ( + "FLOAT", + { + "default": 1.0, + "min": -100.0, + "max": 100.0, + "step": 0.01, + "tooltip": "How strongly to modify the diffusion model. This value can be negative.", + }, + ), + } + } + + RETURN_TYPES = ("MODEL",) + OUTPUT_TOOLTIPS = ("The modified diffusion model.",) + FUNCTION = "load_lora" + TITLE = "Nunchaku Qwen-Image LoRA Loader" + + CATEGORY = "Nunchaku" + DESCRIPTION = ( + "LoRAs are used to modify the diffusion model, " + "altering the way in which latents are denoised such as applying styles. " + "You can link multiple LoRA nodes." + ) + + def load_lora(self, model, lora_name: str, lora_strength: float): + """ + Apply a LoRA to a Nunchaku Qwen-Image diffusion model. + + Parameters + ---------- + model : object + The diffusion model to modify. + lora_name : str + The name of the LoRA to apply. + lora_strength : float + The strength with which to apply the LoRA. + + Returns + ------- + tuple + A tuple containing the modified diffusion model. + """ + if abs(lora_strength) < 1e-5: + return (model,) + + lora_path = folder_paths.get_full_path_or_raise("loras", lora_name) + update_lora_params_v2(model.model.diffusion_model, lora_path, strength=lora_strength) + + logger.info(f"Applied LoRA {lora_name} with strength {lora_strength}") + + return (model,)