Merge pull request #2 from NuttidaLab/tmp

Refactor evaluation code for the spiking network

Author: Sallyliubj

README.md

@@ -18,8 +18,8 @@ The SpikeRNN framework consists of two complementary packages with a modern **ta
 ### Spiking RNN Package
 - Rate-to-spike conversion maintaining task performance
 - Biologically realistic leaky integrate-and-fire (LIF) neurons
-- Spiking task evaluation classes (GoNogoSpikingTask, XORSpikingTask, ManteSpikingTask)
-- SpikingTaskFactory for task-based evaluation
+- Spiking task evaluation classes (GoNogoSpikingEvaluator, XORSpikingEvaluator, ManteSpikingEvaluator)
+- SpikingEvaluatorFactory for task-based evaluation
 - Scaling factor optimization for optimal conversion
 
 ## Task-Based Architecture
@@ -44,13 +44,13 @@ After installation, you can import both packages:
 
 ```python
 from rate import FR_RNN_dale, set_gpu, create_default_config
-from spiking import LIF_network_fnc, lambda_grid_search, evaluate_task
+from spiking import LIF_network_fnc, lambda_grid_search
 
 # Task-based architecture
 from rate import TaskFactory
-from spiking import SpikingTaskFactory
+from spiking.eval_tasks import SpikingEvaluatorFactory
 from rate.tasks import GoNogoTask, XORTask, ManteTask
-from spiking.tasks import GoNogoSpikingTask, XORSpikingTask, ManteSpikingTask
+from spiking.eval_tasks import GoNogoSpikingEvaluator, XORSpikingEvaluator, ManteSpikingEvaluator
 ```
 
 ## Quick Start: Task-Based Architecture
@@ -81,23 +81,22 @@ print(f"Generated {task.__class__.__name__} trial with label: {label}")
 The framework provides 2 levels of evaluation:
 
 ```python
-from spiking import SpikingTaskFactory, evaluate_task
+from spiking.eval_tasks import SpikingEvaluatorFactory, evaluate_task
 
 # Direct task evaluation (when you have a network instance, not necessarily trained)
-spiking_task = SpikingTaskFactory.create_task('go_nogo')
-performance = spiking_task.evaluate_performance(spiking_rnn, n_trials=100)
-print(f"Accuracy: {performance['overall_accuracy']:.2f}")
+spiking_evaluator = SpikingEvaluatorFactory.create_evaluator('go_nogo', settings)
+performance = spiking_evaluator.evaluate_single_trial(model_path, scaling_factor)
+print(f"Trial result: {performance}")
 
 # High-level interface (when you have model files with trained weights)
 performance = evaluate_task(
     task_name='go_nogo',
-    model_dir='models/go-nogo',
-    n_trials=100,
-    save_plots=True
+    model_path='models/go-nogo/model.mat',
+    n_trials=50
 )
 
 # Command line interface (for scripts and automation)
-# python -m spiking.eval_tasks --task go_nogo --model_dir models/go-nogo/
+# python -m spiking.eval_tasks --task go_nogo --model_path models/go-nogo/model.mat
 ```
 
 ### Extending with Custom Tasks
@@ -128,28 +127,32 @@ stimulus, target, label = custom_task.simulate_trial()
 Create custom spiking evaluation tasks:
 
 ```python
-from spiking.tasks import AbstractSpikingTask, SpikingTaskFactory
+from spiking.eval_tasks import SpikingEvaluatorFactory
+from rate.tasks import AbstractTask
 
-class MyCustomSpikingTask(AbstractSpikingTask):
-    def get_default_settings(self):
-        return {'T': 200, 'custom_param': 1.0}
-    
-    def get_sample_trial_types(self):
-        return ['type_a', 'type_b']  # For visualization
+class MyCustomSpikingEvaluator(AbstractTask):
+    def __init__(self, settings):
+        super().__init__(settings)
+        self.eval_amp_thresh = settings.get('eval_amp_thresh', 0.7) # custom value
 
-    def generate_stimulus(self, trial_type=None):
-        # Generate stimulus logic
-        return stimulus, label
+    def validate_settings(self):
+        # Validation logic for custom task
+        required_keys = ['T', 'custom_param']
+        for key in required_keys:
+            if key not in self.settings:
+                raise ValueError(f"Missing required setting: {key}")
 
-    def evaluate_performance(self, spiking_rnn, n_trials=100):
-        # Multi-trial performance evaluation
-        return {'accuracy': 0.85, 'n_trials': n_trials}
+    def evaluate_single_trial(self, model_path: str, scaling_factor: float) -> int:
+        """Evaluate a single trial for the custom task."""
+        # Custom evaluation logic here
+        # Return 1 if correct, 0 if incorrect
+        pass
 
 # Register and use with evaluation system
-SpikingTaskFactory.register_task('my_custom', MyCustomSpikingTask)
+SpikingEvaluatorFactory._registry['my_custom'] = MyCustomSpikingEvaluator
 
 # Now works with eval_tasks.py
-python -m spiking.eval_tasks --task my_custom --model_dir models/custom/
+python -m spiking.eval_tasks --task my_custom --model_path models/custom/model.mat
 ```
 
 ## Requirements
@@ -201,16 +204,20 @@ import numpy as np
 
 # Optimize scaling factor
 lambda_grid_search(
-    model_dir='models/go-nogo',
-    task_name='go-nogo',
+    model_path='models/go-nogo/model.mat',
+    task_name='go_nogo',
     n_trials=100,
-    scaling_factors=list(np.arange(25, 76, 5))
+    scaling_factors=list(np.arange(25, 76, 5)),
+    task_settings=settings
 )
 
 # Evaluate performance
 performance = evaluate_task(
     task_name='go_nogo',
-    model_dir='models/go-nogo/'
+    model_path='models/go-nogo/model.mat',
+    n_trials=50,
+    task_settings=settings,
+    all_trial_types=True
 )
 ```
 

__init__.py

@@ -54,10 +54,6 @@
     LIF_network_fnc,
     create_default_spiking_config,
     lambda_grid_search,
-    SpikingTaskFactory,
-    GoNogoSpikingTask,
-    XORSpikingTask,
-    ManteSpikingTask
 )
 
 def check_packages():
@@ -87,12 +83,6 @@ def check_packages():
     "create_default_spiking_config",
     "lambda_grid_search",
 
-    # Task-based architecture (spiking)
-    "SpikingTaskFactory",
-    "GoNogoSpikingTask",
-    "XORSpikingTask",
-    "ManteSpikingTask",
-    
     # Subpackages
     "rate",
     "spiking",

docs/api/spiking/eval_tasks.rst

@@ -24,7 +24,7 @@ Adapter Classes
 Overview
 ----------------------------------------------------------------------------------
 
-The eval_tasks module provides a high-level evaluation interface that standardizes the process of evaluating trained spiking RNN models across different cognitive tasks. The system is designed to be fully extensible, automatically supporting any task registered with the ``SpikingTaskFactory``.
+The eval_tasks module provides a high-level evaluation interface that standardizes the process of evaluating trained spiking RNN models across different cognitive tasks. The system is designed to be fully extensible, automatically supporting any task registered with the ``SpikingEvaluatorFactory``.
 
 **Key Features:**
 
@@ -35,13 +35,10 @@ The eval_tasks module provides a high-level evaluation interface that standardiz
 * **Robust Error Handling**: Graceful handling of evaluation failures
 * **Flexible Visualization**: Generic visualization system for any task type
 
-**Evaluation Layers:**
+**Evaluation:**
 
-The framework provides three levels of evaluation:
-
-1. **Core Task Methods**: Direct task evaluation (``task.evaluate_performance()``)
-2. **High-Level Interface**: Complete workflow (``evaluate_task()``)
-3. **Command-Line Interface**: Batch processing (``python -m spiking.eval_tasks``)
+1. **High-Level Interface**: Complete workflow (``evaluate_task()``)
+2. **Command-Line Interface**: Batch processing (``python -m spiking.eval_tasks``)
 
 Usage Examples
 ----------------------------------------------------------------------------------
@@ -54,49 +51,52 @@ Usage Examples
     
     # Evaluate any registered task
     performance = evaluate_task(
-        task_name='go_nogo',           # or 'xor', 'mante', custom tasks
-        model_dir='models/go-nogo/',
-        save_plots=True
+        task_name='go_nogo',
+        model_path='models/go-nogo/model.mat',
+        n_trials=50
     )
     
-    print(f"Accuracy: {performance['overall_accuracy']:.3f}")
+    print(f"Performance: {performance}")
 
 **Command-Line Interface:**
 
 .. code-block:: bash
 
     # Basic evaluation
-    python -m spiking.eval_tasks --task go_nogo --model_dir models/go-nogo/
+    python -m spiking.eval_tasks --task go_nogo --model_path models/go-nogo/model.mat
     
     # With custom parameters
     python -m spiking.eval_tasks \
         --task xor \
-        --model_dir models/xor/ \
+        --model_path models/xor/model.mat \
         --scaling_factor 45.0 \
-        --no_plots
+        --n_trials 50
     
     # Custom task (after registration)
-    python -m spiking.eval_tasks --task my_custom --model_dir models/custom/
+    python -m spiking.eval_tasks --task my_custom --model_path models/custom/model.mat
 
 **Custom Task Integration:**
 
 .. code-block:: python
 
-    from spiking.tasks import SpikingTaskFactory, AbstractSpikingTask
-    from spiking.eval_tasks import evaluate_task
-    
-    # 1. Define custom task
-    class WorkingMemoryTask(AbstractSpikingTask):
-        # ... implementation ...
-        pass
-    
+    from spiking.eval_tasks import SpikingEvaluatorFactory, evaluate_task
+    from rate.tasks import AbstractTask
+
+    # 1. Define custom evaluator (inheriting from a rate task class)
+    class WorkingMemoryEvaluator(AbstractTask):
+        def validate_settings(self):
+            pass
+        def evaluate_single_trial(self, model_path, scaling_factor, model_data=None):
+            # ... implementation ...
+            pass
+
     # 2. Register with factory
-    SpikingTaskFactory.register_task('working_memory', WorkingMemoryTask)
-    
+    SpikingEvaluatorFactory._registry['working_memory'] = WorkingMemoryEvaluator
+
     # 3. Evaluate using unified interface
     performance = evaluate_task(
-        task_name='working_memory',  # Now supported automatically
-        model_dir='models/working_memory/',
+        task_name='working_memory',
+        model_path='models/working_memory/model.mat',
     )
 
 Command-Line Arguments
@@ -108,22 +108,26 @@ Command-Line Arguments
 
    Task to evaluate. Available tasks are dynamically determined from the factory registry.
 
-.. option:: --model_dir MODEL_DIR
+.. option:: --model_path MODEL_PATH
 
-   Directory containing the trained model .mat file.
+   Path to the trained model .mat file.
 
 .. option:: --scaling_factor SCALING_FACTOR
 
    Override scaling factor (uses value from .mat file if not provided).
 
-.. option:: --no_plots
+.. option:: --n_trials N_TRIALS
 
-   Skip generating visualization plots.
+   Number of trials to evaluate.
 
 .. option:: --T T
 
    Trial duration (timesteps) - overrides task default.
 
+.. option:: --delay DELAY
+
+   Delay time (timesteps) - overrides task default.
+
 .. option:: --stim_on STIM_ON
 
    Stimulus onset time - overrides task default.
@@ -141,21 +145,4 @@ The system automatically loads trained rate RNN models from `.mat` files and ext
 
 * Network weights and connectivity matrices
 * Optimal scaling factors for rate-to-spike conversion
-* Task-specific parameters and configurations
-
-**Generic Visualization:**
-
-The visualization system uses each task's ``get_sample_trial_types()`` method to determine what trial types to generate for plotting. This allows custom tasks to specify their own visualization patterns without modifying the evaluation code.
-
-**Error Handling:**
-
-The evaluation system includes comprehensive error handling:
-
-* Graceful handling of missing model files
-* Validation of task names against factory registry
-* Recovery from trial generation failures
-* Informative error messages for debugging
-
-**Extensibility:**
-
-The system is designed to be fully extensible. Any task that inherits from ``AbstractSpikingTask`` and is registered with ``SpikingTaskFactory`` can be evaluated using this unified interface.
+* Task-specific parameters and configurations

docs/api/spiking/index.rst

@@ -11,7 +11,6 @@ Core Modules
    :maxdepth: 1
 
    lif_network
-   tasks
    eval_tasks
    lambda_grid_search
    utils
@@ -22,9 +21,6 @@ Module Overview
 **LIF_network_fnc.py**
     Core function for converting rate RNNs to spiking networks and running LIF simulations.
 
-**tasks.py**
-    Task-based architecture for spiking neural network evaluation with abstract base classes and concrete task implementations.
-
 **eval_tasks.py**
     Unified, extensible evaluation interface for spiking neural networks on cognitive tasks.
 
@@ -43,14 +39,6 @@ Quick Reference
 * ``evaluate_task()``: Unified evaluation interface for all tasks
 * ``lambda_grid_search()``: Optimize scaling factors
 
-**Task Classes:**
-
-* ``AbstractSpikingTask``: Base class for spiking task evaluation
-* ``GoNogoSpikingTask``: Go-NoGo task for spiking networks
-* ``XORSpikingTask``: XOR task for spiking networks
-* ``ManteSpikingTask``: Mante task for spiking networks
-* ``SpikingTaskFactory``: Factory for creating spiking task instances
-
 **Configuration:**
 
 * ``SpikingConfig``: Configuration dataclass for spiking RNN parameters
@@ -60,4 +48,3 @@ Quick Reference
 
 * ``load_rate_model()``: Load rate model from `.mat` file
 * ``format_spike_data()``: Format spike data for analysis
-* ``SpikingTaskFactory.register_task()``: Register custom tasks