debug_generation.py

#!/usr/bin/env python3
"""
Debug script to test generation parameters step by step.
"""

import sys
from pathlib import Path

# Add src to path
sys.path.append(str(Path(__file__).parent / "src"))

from models.model_factory import ModelFactory
from adaptive.adaptive_cot import AdaptiveCoT

def test_generation_parameters():
    """Test generation parameters step by step."""
    print("🔬 Debug Generation Parameters")
    print("=" * 60)
    
    # Load model
    print("🔧 Loading model...")
    model = ModelFactory.create_model(
        model_type="deepseek",
        model_name="/raid/LLM/llama3.1-8b-instruct",
        config={"gpu_id": 0}
    )
    model.load_model()
    
    # Test problem
    problem = "Janet's ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?"
    
    print(f"Problem: {problem}")
    print()
    
    # Test 1: Direct generation with exact parameters
    print("🔧 Test 1: Direct Generation")
    print("-" * 40)
    
    prompt = f"Q: {problem}\nA:"
    print(f"Prompt: {prompt}")
    print()
    
    generated = model.generate(
        prompt,
        max_tokens=512,
        temperature=0.0,
        top_p=1.0,
        do_sample=False,
        num_return_sequences=1
    )
    
    print(f"Generated: {generated}")
    print()
    
    # Test 2: Our framework with exact same parameters
    print("🔧 Test 2: Our Framework")
    print("-" * 40)
    
    config = {
        "adaptive_branching": False,
        "min_branches": 1,
        "max_branches": 1,
        "default_branches": 1,
        "num_fewshot": 0,
        "temperature": 0.0,
        "top_p": 1.0,
        "max_tokens": 512,
    }
    
    adaptive_cot = AdaptiveCoT(model, config)
    
    # Test the generation method directly
    reasoning_paths = adaptive_cot._generate_reasoning_paths(problem, 1, {})
    
    print(f"Generated: {reasoning_paths[0] if reasoning_paths else 'None'}")
    print()
    
    # Test 3: Check if they're identical
    print("🔧 Test 3: Comparison")
    print("-" * 40)
    
    direct_text = generated
    framework_text = reasoning_paths[0] if reasoning_paths else ""
    
    print(f"Direct length: {len(direct_text)}")
    print(f"Framework length: {len(framework_text)}")
    print(f"Identical: {direct_text == framework_text}")
    
    if direct_text != framework_text:
        print("\nFirst 200 chars of direct:")
        print(direct_text[:200])
        print("\nFirst 200 chars of framework:")
        print(framework_text[:200])
    
    # Test 4: Check generation parameters being passed
    print("\n🔧 Test 4: Generation Parameters")
    print("-" * 40)
    
    # Override the generate method to log parameters
    original_generate = model.generate
    
    def logged_generate(prompt, **kwargs):
        print(f"Parameters passed to model.generate:")
        for key, value in kwargs.items():
            print(f"  {key}: {value}")
        print()
        return original_generate(prompt, **kwargs)
    
    model.generate = logged_generate
    
    # Test direct generation again
    print("Direct generation parameters:")
    generated = model.generate(
        prompt,
        max_tokens=512,
        temperature=0.0,
        top_p=1.0,
        do_sample=False,
        num_return_sequences=1
    )
    
    # Test framework generation again
    print("\nFramework generation parameters:")
    reasoning_paths = adaptive_cot._generate_reasoning_paths(problem, 1, {})

if __name__ == "__main__":
    test_generation_parameters()