added documentation

Author: robotastic

.gitignore

@@ -8,7 +8,6 @@
 *.yaml
 *.txt
 *.json
-*.png
 *.mdb
 *-checkpoint.ipynb
 .DS_Store

poetry.lock

@@ -10,14 +10,16 @@ python = "^3.10"
 scikit-image = "^0.22.0"
 matplotlib = "^3.8.3"
 numpy = "^1.26.4"
-opencv-python = "^4.9.0.80"
+opencv-python = "^4.8.0.74"
 scipy = "^1.12.0"
 zstandard = "^0.22.0"
 pyyaml = "^6.0.1"
 pillow = "^10.2.0"
 python-on-whales = "^0.69.0"
 sigmf = "^1.2.0"
-
+tqdm = "^4.66.4"
+cupy = "^13.2.0"
+torchsig = {path = "../torchsig"}
 
 [tool.poetry.group.dev.dependencies]
 jupyter = "^1.0.0"

pyproject.toml

@@ -1,3 +1,74 @@
+# Preqs
+
+## Poetry
+
+## Torchsig
+
+Download and install from [Github](https://github.com/TorchDSP/torchsig)
+
+```
+git clone https://github.com/TorchDSP/torchsig.git
+cd torchsig
+sudo pip install .
+```
+
+
+
+# Building a Model
+
+
+## Approach
+
+Our current approach is to capture samples of the background RF environment and then also isolate signals of interest and capture samples of each of the signals. The same label will be applied to all of the signals present in the background environment samples. We use this to essentially teach the model to ignore those signals. For this to work, it is important that none of the signals of interest are present. Since it is really tough these days to find an RF free environment, we have build a mini-faraday cage enclosure by lining the inside of a pelican case with foil. There are lots of instructions, like [this one](https://mosequipment.com/blogs/blog/build-your-own-faraday-cage), available online if you want to build your own. With this, the signal will be very strong, so make sure you adjust the SDR's gain appropriately.
+
+## Labeling IQ Data
+
+The scripts in the [label_scripts](./label_scripts/) use signal processing to automatically label IQ data. The scripts looks at the signal power to detect when there is a signal present in the IQ data. When a signal is detected, the script will look at the frequencies for that set of samples and find the upper and lower bounds.
+
+
+
+
+### Annotation Explained
+
+```python
+annotation_utils.annotate(
+                f, 
+                label="mavic3_video",               # This is the label that is applied to all of the matching annotations
+                avg_window_len=256,                 # The number of samples over which to average signal power
+                avg_duration=0.25,                  # The number of seconds, from the start of the recording to use to automatically calculate the SNR threshold, if it is None then all of the samples will be used
+                debug=False,    
+                estimate_frequency=True,            # Whether the frequency bounds for an annotation should be calculated. estimate_frequency needs to be enabled if you use min/max_bandwidth
+                spectral_energy_threshold=0.95,     # Percentage used to determine the upper and lower frequency bounds for an annotation
+                force_threshold_db=-58,             # Used to manually set the threshold used for detecting a signal and creating an annotation. If None, then the automatic threshold calcuation will be used instead.
+                overwrite=False,                    # If True, any existing annotations in the .sigmf-meta file will be removed
+                min_bandwidth=16e6,                 # The minimum bandwidth (in Hz) of a signal to annotate
+                max_bandwidth=None,                 # The maximum bandwidth (in Hz) of a signal to annotate
+                min_annotation_length=10000,        # The minimum numbers of samples in length a signal needs to be in order for it to be annotated. This is directly related to the sample rate a signal was captured at and does not take into account bandwidth. So 10000 samples at 20,000,000 samples per second, would mean a minimum transmission length of 0.0005 seconds
+                # max_annotations=500,              # The maximum number of annotations to automatically add  
+                dc_block=True                       # De-emphasize the DC spike when trying to calculate the frequencies for a signal
+            )
+```
+
+### Configuring Annotation
+
+#### Force Threshold dB
+![low threshold](./images/low_threshold.png)
+
+If you see annotations where harmonics or lower power, unintentional signals are getting selected, try setting the `force_threshold_db`. The automatic threshold calculation maybe selecting a value that is too low. Find a value for `force_threshold_db` where it is selecting the intended signals and ignoring the low power ones.
+
+#### Spectral Energy Threshold
+![spectral energy](./images/spectral_energy.png)
+
+If the frequency bounds are not lining up with the top or bottom part of a signal, make the `spectral_energy_threshold` higher. Sometime a setting as high as 0.99 is required 
+
+#### Skipping "small" Signals
+![small signals](./images/min_annotation.png)
+
+Some tuning is needed for signals that have a short transmission duration and/or limited bandwidth. Here are a couple things to try if they are getting skipped:
+- `min_annotation_length` is the minimum number of samples for an annotation. If the signal is has less samples than this, it will not be annotated. Try lowering this.
+- The `average_duration` setting maybe too long and the signal is getting averaged into the noise. Try lowering this.
+- `min_bandwidth` is the minimum bandwidth (in Hz) for a signal to be detected. If this value is too high, signals that have less bandiwdth will be ignored. Try lowering this.
+
 
 ## Files
 

rfml-dev/README.md

@@ -13,13 +13,13 @@
 from pathlib import Path
 from tqdm import tqdm
 
-def moving_average(complex_iq, window_len):
+def moving_average(complex_iq, avg_window_len):
     return (
-        np.convolve(np.abs(complex_iq) ** 2, np.ones(window_len), "valid") / window_len
+        np.convolve(np.abs(complex_iq) ** 2, np.ones(avg_window_len), "valid") / avg_window_len
     )
 
-def power_squelch(iq_samples, threshold, window):
-    avg_pwr = moving_average(iq_samples, window)
+def power_squelch(iq_samples, threshold, avg_window_len):
+    avg_pwr = moving_average(iq_samples, avg_window_len)
     avg_pwr_db = 10 * np.log10(avg_pwr)
 
     good_samples = np.zeros(len(iq_samples))
@@ -39,11 +39,12 @@ def reset_annotations(data_obj):
 
 def annotate_power_squelch(data_obj, threshold, avg_window_len, label=None, skip_validate=False, estimate_frequency=False, dry_run=False, min_annotation_length=400, spectral_energy_threshold=None, min_bandwidth=None, max_bandwidth=None, overwrite=True, max_annotations=None, dc_block=False, verbose=False):
     iq_samples = data_obj.get_samples()
-    idx = power_squelch(iq_samples, threshold=threshold, window=avg_window_len)
+    idx = power_squelch(iq_samples, threshold=threshold, avg_window_len=avg_window_len)
 
     if overwrite:
         data_obj.sigmf_obj._metadata[data_obj.sigmf_obj.ANNOTATION_KEY] = []
     for start, stop in tqdm(idx[:max_annotations]):
+        #print(f"{start=}, {stop=}  {max_annotations=}")
         start, stop = int(start), int(stop)
         if min_annotation_length and (stop-start < min_annotation_length):
             continue
@@ -52,10 +53,13 @@ def annotate_power_squelch(data_obj, threshold, avg_window_len, label=None, skip
             freq_lower_edge, freq_upper_edge = get_occupied_bandwidth(iq_samples[start:stop], data_obj.metadata["global"]["core:sample_rate"], data_obj.metadata["captures"][0]["core:frequency"], spectral_energy_threshold=spectral_energy_threshold, dc_block=dc_block, verbose=verbose)
             bandwidth = freq_upper_edge - freq_lower_edge
             if min_bandwidth and bandwidth < min_bandwidth: 
+                if verbose:
+                    print(f"min_bandwidth - Skipping, {label}, {start=}, {stop=}, {bandwidth=}, {freq_upper_edge=}, {freq_lower_edge=}")
                 # print(f"Skipping, {label}, {start=}, {stop=}, {bandwidth=}, {freq_upper_edge=}, {freq_lower_edge=}")
                 continue
             if max_bandwidth and bandwidth > max_bandwidth: 
-                # print(f"Skipping, {label}, {start=}, {stop=}, {bandwidth=}, {freq_upper_edge=}, {freq_lower_edge=}")
+                if verbose:
+                    print(f"max_bandwidth - Skipping, {label}, {start=}, {stop=}, {bandwidth=}, {freq_upper_edge=}, {freq_lower_edge=}")
                 continue
 
         else: 
@@ -83,57 +87,58 @@ def annotate(filename, label, avg_window_len, avg_duration=-1, debug=False, dry_
 
     data_obj = data_class.Data(filename)
 
-    # use a seconds worth of data to calculate threshold
-    if avg_duration > -1:
-        iq_samples = data_obj.get_samples(n_samples=int(data_obj.metadata["global"]["core:sample_rate"]*avg_duration))
-        if iq_samples is None: 
-            iq_samples = data_obj.get_samples()
+
+    if force_threshold_db:
+        threshold_db = force_threshold_db
     else:
-        iq_samples = data_obj.get_samples()
-    
-    avg_pwr = moving_average(iq_samples, avg_window_len)
-    avg_pwr_db = 10*np.log10(avg_pwr)
-    del(avg_pwr)
-    del(iq_samples)
+        # use a seconds worth of data to calculate threshold
+        if avg_duration > -1:
+            iq_samples = data_obj.get_samples(n_samples=int(data_obj.metadata["global"]["core:sample_rate"]*avg_duration))
+            if iq_samples is None: 
+                iq_samples = data_obj.get_samples()
+        else:
+            iq_samples = data_obj.get_samples()
+        
+        avg_pwr = moving_average(iq_samples, avg_window_len)
+        avg_pwr_db = 10*np.log10(avg_pwr)
+        del(avg_pwr)
+        del(iq_samples)
 
 
-    # current threshold in custom_handler 
-    guess_threshold_old = (np.max(avg_pwr_db) + np.mean(avg_pwr_db))/2
+        # current threshold in custom_handler 
+        guess_threshold_old = (np.max(avg_pwr_db) + np.mean(avg_pwr_db))/2
 
-    # MAD estimator
-    def median_absolute_deviation(series):
-        mad = 1.4826 * np.median(np.abs(series - np.median(series)))
-        # sci_mad = scipy.stats.median_abs_deviation(series, scale="normal")
-        return np.median(series) + 6*mad
+        # MAD estimator
+        def median_absolute_deviation(series):
+            mad = 1.4826 * np.median(np.abs(series - np.median(series)))
+            # sci_mad = scipy.stats.median_abs_deviation(series, scale="normal")
+            return np.median(series) + 6*mad
 
-    mad = median_absolute_deviation(avg_pwr_db)
+        mad = median_absolute_deviation(avg_pwr_db)
 
-    if force_threshold_db:
-        threshold_db = force_threshold_db
-    else:
         threshold_db = mad
 
-    if debug:
-        print(f"{np.max(avg_pwr_db)=}")
-        print(f"{np.mean(avg_pwr_db)=}")
-        print(f"median absolute deviation threshold = {mad}")
-        print(f"using threshold = {threshold_db}")
-        # print(f"{len(avg_pwr_db)=}")
-        
-        plt.figure()
-        db_plot = avg_pwr_db[int(0*20.48e6):int(avg_duration*20.48e6)]
-        plt.plot(np.arange(len(db_plot))/data_obj.metadata["global"]["core:sample_rate"], db_plot)
-        plt.axhline(y = guess_threshold_old, color = 'g', linestyle = '-', label="old threshold") 
-        plt.axhline(y = np.mean(avg_pwr_db), color = 'r', linestyle = '-', label="average") 
-        plt.axhline(y = mad, color = 'b', linestyle = '-', label="median absolute deviation threshold") 
-        if force_threshold_db:
-            plt.axhline(y = force_threshold_db, color = 'yellow', linestyle = '-', label="force threshold db") 
-        plt.legend(loc="upper left")
-        plt.ylabel("dB")
-        plt.xlabel("time (seconds)")
-        plt.title("Signal Power")
-        plt.show()
-
+        if debug:
+            print(f"{np.max(avg_pwr_db)=}")
+            print(f"{np.mean(avg_pwr_db)=}")
+            print(f"median absolute deviation threshold = {mad}")
+            print(f"using threshold = {threshold_db}")
+            # print(f"{len(avg_pwr_db)=}")
+            
+            plt.figure()
+            db_plot = avg_pwr_db[int(0*20.48e6):int(avg_duration*20.48e6)]
+            plt.plot(np.arange(len(db_plot))/data_obj.metadata["global"]["core:sample_rate"], db_plot)
+            plt.axhline(y = guess_threshold_old, color = 'g', linestyle = '-', label="old threshold") 
+            plt.axhline(y = np.mean(avg_pwr_db), color = 'r', linestyle = '-', label="average") 
+            plt.axhline(y = mad, color = 'b', linestyle = '-', label="median absolute deviation threshold") 
+            if force_threshold_db:
+                plt.axhline(y = force_threshold_db, color = 'yellow', linestyle = '-', label="force threshold db") 
+            plt.legend(loc="upper left")
+            plt.ylabel("dB")
+            plt.xlabel("time (seconds)")
+            plt.title("Signal Power")
+            plt.show()
+    print(f"Using dB threshold = {threshold_db} for detecting signals to annotate")
     annotate_power_squelch(data_obj, threshold_db, avg_window_len, label=label, skip_validate=True, estimate_frequency=estimate_frequency, spectral_energy_threshold=spectral_energy_threshold, min_bandwidth=min_bandwidth, max_bandwidth=max_bandwidth,  dry_run=dry_run, min_annotation_length=min_annotation_length, overwrite=overwrite, max_annotations=max_annotations, dc_block=dc_block, verbose=verbose)
 
 def get_occupied_bandwidth(samples, sample_rate, center_frequency, spectral_energy_threshold=None, dc_block=False, verbose=False):
@@ -174,6 +179,9 @@ def get_occupied_bandwidth(samples, sample_rate, center_frequency, spectral_ener
         if freq_power_normalized[lower_idx:upper_idx].sum() >= spectral_energy_threshold:
             break
 
+        if lower_idx == 0 and upper_idx == freq_power_normalized.shape[0]-1:
+            print(f"Could not find spectral energy threshold - max was: {freq_power_normalized[lower_idx:upper_idx].sum()}")
+            break
 
     freq_upper_edge = center_frequency - (freq_power.shape[0]/2 - upper_idx)/freq_power.shape[0]*sample_rate
     freq_lower_edge = center_frequency - (freq_power.shape[0]/2 - lower_idx)/freq_power.shape[0]*sample_rate

rfml-dev/annotation_utils.py

@@ -2,19 +2,18 @@
 
 from pathlib import Path
 from tqdm import tqdm
-
+import os
+import sys
+sys.path.insert(1, os.path.join(sys.path[0], '..'))
 import annotation_utils
 import data as data_class
 
 
 
 data_globs = {
-    # "mini2_video": [
-    #     "data/gamutrf/gamutrf-arl/01_30_23/mini2_iq_label/*.sigmf-meta",
-    # ],
-    "mini2_video": [
-        # "data/gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/test_iq_label/dji-mini2-200m-0deg-5735mhz-lp-50-gain_20p5Msps_craft_flying-1.raw.sigmf-meta",
-        "data/gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/test_iq_label/*.sigmf-meta",
+    "dji_samples": [
+        "/home/iqt/lberndt/gamutrf-depoly/data/samples/mavic-30db/samples_1722867361.666000_2408703998Hz_20480000sps.raw.sigmf-meta",
+        "/home/iqt/lberndt/gamutrf-depoly/data/samples/mavic-0db/samples_1722883251.180000_2408703998Hz_20480000sps.raw.sigmf-meta"
     ]
 }
 
@@ -29,30 +28,31 @@
             annotation_utils.reset_annotations(data_obj)
             annotation_utils.annotate(
                 f, 
-                label="mini2_video", 
+                label="mavic3_remoteid", 
                 avg_window_len=256, 
-                avg_duration=0.25, 
+                avg_duration=0.10, 
                 debug=False, 
                 estimate_frequency=True, 
-                spectral_energy_threshold=0.95, 
-                force_threshold_db=-58, 
-                overwrite=False, 
-                min_bandwidth=16e6, 
-                min_annotation_length=10000, 
+                spectral_energy_threshold=0.90, 
+                #force_threshold_db=-48, 
+                overwrite=True, 
+                min_bandwidth=1e5, 
+                max_bandwidth=2e6,
+                min_annotation_length=500, 
                 # max_annotations=500, 
                 dc_block=True
             )
             annotation_utils.annotate(
                 f, 
-                label="mini2_telem",  
+                label="mavic3_video",  
                 avg_window_len=256, 
                 avg_duration=0.25, 
                 debug=False, 
                 estimate_frequency=True, 
-                spectral_energy_threshold=0.95, 
-                force_threshold_db=-58, 
+                spectral_energy_threshold=0.99, 
+                force_threshold_db=-38, 
                 overwrite=False, 
-                max_bandwidth=16e6, 
+                min_bandwidth=8e6, 
                 min_annotation_length=10000, 
                 # max_annotations=500, 
                 dc_block=True

rfml-dev/images/low_threshold.png

@@ -0,0 +1,49 @@
+import glob
+
+from pathlib import Path
+from tqdm import tqdm
+import os
+import sys
+sys.path.insert(1, os.path.join(sys.path[0], '..'))
+import annotation_utils
+import data as data_class
+
+
+
+data_globs = {
+    # "mini2_video": [
+    #     "data/gamutrf/gamutrf-arl/01_30_23/mini2_iq_label/*.sigmf-meta",
+    # ],
+    "mini2_video": [
+        "/home/iqt/lberndt/gamutrf-depoly/data/samples/environment/samples_1722872733.648000_2408703998Hz_20480000sps.raw.sigmf-meta"
+        #"/home/iqt/lberndt/gamutrf-depoly/data/samples/mavic-0db/samples_1722883251.180000_2408703998Hz_20480000sps.raw.sigmf-meta"
+    ]
+}
+
+
+
+for label in data_globs:
+    for data_glob in data_globs[label]:
+        for f in tqdm(glob.glob(str(Path(data_glob)))):
+            # annotation_utils.annotate(f, label=label, avg_window_len=256, avg_duration=0.25, debug=True, estimate_frequency=True, spectral_energy_threshold=0.99, force_threshold_db=-40)
+
+            data_obj = data_class.Data(f)
+            annotation_utils.reset_annotations(data_obj)
+            annotation_utils.annotate(
+                f, 
+                label="environment", 
+                avg_window_len=256, 
+                avg_duration=0.10, 
+                debug=False, 
+                estimate_frequency=True, 
+                spectral_energy_threshold=0.90, 
+                #force_threshold_db=-48, 
+                overwrite=True, 
+                min_bandwidth=None, 
+                max_bandwidth=None, 
+                min_annotation_length=1000, 
+                # max_annotations=500, 
+                dc_block=True
+            )
+
+            

rfml-dev/images/min_annotation.png

@@ -10,129 +10,129 @@
 experiments = {
     "experiment_0": {
         "experiment_name": "experiment_0",
-        "class_list": ["wifi","anom_wifi"],
-        "train_dir": ["data/gamutrf/gamutrf-sd-gr-ieee-wifi/test_offline"],
+        "class_list": ["mavic3_video","mavic3_remoteid","env"],
+        "train_dir": ["/home/iqt/lberndt/gamutrf-depoly/data/samples/mavic-30db", "/home/iqt/lberndt/gamutrf-depoly/data/samples/mavic-0db", "/home/iqt/lberndt/gamutrf-depoly/data/samples/environment"],
         "iq_epochs": 10,
         "spec_epochs": 10,
-        "notes": "TESTING"
-    },
-    "experiment_1": {
-        "experiment_name": "experiment_1",
-        "class_list": ["wifi","anom_wifi"],
-        "train_dir": ["data/gamutrf/gamutrf-nz-anon-wifi", "data/gamutrf/gamutrf-nz-nonanon-wifi"],
-        "iq_epochs": 40,
-        "spec_epochs": 40,
-        "notes": "Wi-Fi vs anomalous Wi-Fi, Ettus B200Mini, anarkiwi collect"
-    },
-    "experiment_2": {
-        "experiment_name": "experiment_2",
-        "class_list": ["mini2_video","mini2_telem"],
-        "train_dir": ["dev_data/torchsig_train/samples"],
-        "iq_epochs": 25,
-        "spec_epochs": 50,
-        "notes": "DJI Mini2, Ettus B200Mini RX, copy of lab collection gamutrf/gamutrf-arl/01_30_23/mini2"
-    },
-    "experiment_3": {
-        "experiment_name": "experiment_3",
-        "class_list": ["wifi","anom_wifi"],
-        "train_dir": ["data/gamutrf/gamutrf-nz-anon-wifi", "data/gamutrf/gamutrf-nz-nonanon-wifi"],
-        "val_dir": ["data/gamutrf/gamutrf-wifi-and-anom-bladerf"],
-        "iq_epochs": 40,
-        "spec_epochs": 70,
-        "spec_skip_export": True, # USE WITH CAUTION (but speeds up large directories significantly): skip after first run if using separate train/val directories 
-        "notes": "Wi-Fi vs anomalous Wi-Fi, train on Ettus B200Mini RX/TX, validate on BladeRF TX & Ettus B200Mini RX, anarkiwi collect"
-    },
-    "experiment_4": {
-        "experiment_name": "experiment_4",
-        "class_list": ["wifi","anom_wifi"],
-        "train_dir": ["data/gamutrf/gamutrf-nz-anon-wifi", "data/gamutrf/gamutrf-nz-nonanon-wifi"],
-        "val_dir": ["data/gamutrf/gamutrf-nz-wifi"],
-        "iq_epochs": 40,
-        "spec_epochs": 100,
-        "notes": "Wi-Fi vs anomalous Wi-Fi, train on Ettus B200Mini RX/TX, validate on real Wi-Fi AP TX & Ettus B200Mini RX, anarkiwi collect"
-    },
-    "experiment_5": {
-        "experiment_name": "experiment_5",
-        "class_list": ["mini2_video","mini2_telem"],
-        "train_dir": ["dev_data/torchsig_train/samples"],
-        "val_dir": ["data/gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/test_iq_label"],
-        "iq_epochs": 40,
-        "spec_epochs": 100,
-        "spec_force_yolo_label_larger": True,
-        "notes": "DJI Mini2, Ettus B200Mini RX, train on copy of lab collection gamutrf/gamutrf-arl/01_30_23/mini2, validate on field collect gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/test_iq_label"
-    },
-    "experiment_6": {
-        "experiment_name": "experiment_6",
-        "class_list": ["wifi","anom_wifi"],
-        "train_dir": [
-            "data/gamutrf/wifi-data-03082024/20msps/normal/train",
-            "data/gamutrf/wifi-data-03082024/20msps/normal/test",
-            "data/gamutrf/wifi-data-03082024/20msps/normal/inference",
-            "data/gamutrf/wifi-data-03082024/20msps/mod/train",
-            "data/gamutrf/wifi-data-03082024/20msps/mod/test",
-            "data/gamutrf/wifi-data-03082024/20msps/mod/inference",
-        ],
-        "val_dir": [
-            "data/gamutrf/wifi-data-03082024/20msps/normal/validate",
-            "data/gamutrf/wifi-data-03082024/20msps/mod/validate",
-        ], 
-        "iq_num_samples": 16*25, 
-        "iq_epochs": 10,
-        "iq_batch_size": 16,
-        "spec_batch_size": 32,
-        "spec_epochs": 40,
-        "spec_n_fft": 16,
-        "spec_time_dim": 25,
-        "notes": "Ettus B200Mini RX, emair collect"
-    },
-    "experiment_7": {
-        "experiment_name": "experiment_7",
-        "class_list": ["wifi","anom_wifi"],
-        "train_dir": ["data/gamutrf/gamutrf-wifi-and-anom-bladerf"],
-        "iq_epochs": 40,
-        "spec_epochs": 40,
-        "notes": "Wi-Fi vs anomalous Wi-Fi, BladeRF, anarkiwi collect"
-    },
-    "experiment_8": {
-        "experiment_name": "experiment_8",
-        "class_list": ["wifi","anom_wifi"],
-        "train_dir": ["data/gamutrf/gamutrf-wifi-and-anom-bladerf"],
-        "val_dir": ["data/gamutrf/gamutrf-nz-anon-wifi", "data/gamutrf/gamutrf-nz-nonanon-wifi"],
-        "iq_epochs": 40,
-        "spec_epochs": 70,
-        "spec_skip_export": True, # USE WITH CAUTION (but speeds up large directories significantly): skip after first run if using separate train/val directories 
-        "notes": "Wi-Fi vs anomalous Wi-Fi, validate on BladeRF TX & Ettus B200Mini RX, train on Ettus B200Mini RX/TX, anarkiwi collect"
-    },
-    "experiment_9": {
-        "experiment_name": "experiment_9",
-        "class_list": ["mini2_video","mini2_telem"],
-        "train_dir": ["dev_data/torchsig_train/samples"],
-        "val_dir": ["data/gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/test_iq_label"],
-        "iq_epochs": 40,
-        "spec_epochs": 200,
-        "spec_yolo_augment": True,
-        "notes": "DJI Mini2, Ettus B200Mini RX, train on copy of lab collection gamutrf/gamutrf-arl/01_30_23/mini2, validate on field collect gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/test_iq_label"
-    },
-    "experiment_10": {
-        "experiment_name": "experiment_10",
-        "class_list": ["wifi","anom_wifi"],
-        "train_dir": ["data/gamutrf/anom_wifi/train_gamutrf-nz-anon-wifi", "data/gamutrf/anom_wifi/train_gamutrf-nz-nonanon-wifi", "data/gamutrf/anom_wifi/train_gamutrf-wifi-and-anom-bladerf"],
-        "val_dir": ["data/gamutrf/anom_wifi/val_gamutrf-nz-anon-wifi", "data/gamutrf/anom_wifi/val_gamutrf-nz-nonanon-wifi"],
-        "iq_epochs": 40,
-        "spec_epochs": 70,
-        "spec_skip_export": True, # USE WITH CAUTION (but speeds up large directories significantly): skip after first run if using separate train/val directories 
-        "notes": "Wi-Fi vs anomalous Wi-Fi, train on both BladeRF TX & Ettus B200Mini RX and on Ettus B200Mini RX/TX, validate on Ettus B200Mini RX/TX, anarkiwi collect"
-    },
-    "experiment_11": {
-        "experiment_name": "experiment_11",
-        "class_list": ["wifi","anom_wifi"],
-        "train_dir": ["data/gamutrf/anom_wifi/train_gamutrf-nz-anon-wifi", "data/gamutrf/anom_wifi/train_gamutrf-nz-nonanon-wifi", "data/gamutrf/anom_wifi/train_gamutrf-wifi-and-anom-bladerf"],
-        "val_dir": ["data/gamutrf/anom_wifi/val_gamutrf-wifi-and-anom-bladerf"],
-        "iq_epochs": 40,
-        "spec_epochs": 70,
-        "spec_skip_export": True, # USE WITH CAUTION (but speeds up large directories significantly): skip after first run if using separate train/val directories 
-        "notes": "Wi-Fi vs anomalous Wi-Fi, train on both BladeRF TX & Ettus B200Mini RX and on Ettus B200Mini RX/TX, validate on BladeRF TX, anarkiwi collect"
+        "notes": "DJI Mavic3 Detection"
     },
+    # "experiment_1": {
+    #     "experiment_name": "experiment_1",
+    #     "class_list": ["wifi","anom_wifi"],
+    #     "train_dir": ["data/gamutrf/gamutrf-nz-anon-wifi", "data/gamutrf/gamutrf-nz-nonanon-wifi"],
+    #     "iq_epochs": 40,
+    #     "spec_epochs": 40,
+    #     "notes": "Wi-Fi vs anomalous Wi-Fi, Ettus B200Mini, anarkiwi collect"
+    # },
+    # "experiment_2": {
+    #     "experiment_name": "experiment_2",
+    #     "class_list": ["mini2_video","mini2_telem"],
+    #     "train_dir": ["dev_data/torchsig_train/samples"],
+    #     "iq_epochs": 25,
+    #     "spec_epochs": 50,
+    #     "notes": "DJI Mini2, Ettus B200Mini RX, copy of lab collection gamutrf/gamutrf-arl/01_30_23/mini2"
+    # },
+    # "experiment_3": {
+    #     "experiment_name": "experiment_3",
+    #     "class_list": ["wifi","anom_wifi"],
+    #     "train_dir": ["data/gamutrf/gamutrf-nz-anon-wifi", "data/gamutrf/gamutrf-nz-nonanon-wifi"],
+    #     "val_dir": ["data/gamutrf/gamutrf-wifi-and-anom-bladerf"],
+    #     "iq_epochs": 40,
+    #     "spec_epochs": 70,
+    #     "spec_skip_export": True, # USE WITH CAUTION (but speeds up large directories significantly): skip after first run if using separate train/val directories 
+    #     "notes": "Wi-Fi vs anomalous Wi-Fi, train on Ettus B200Mini RX/TX, validate on BladeRF TX & Ettus B200Mini RX, anarkiwi collect"
+    # },
+    # "experiment_4": {
+    #     "experiment_name": "experiment_4",
+    #     "class_list": ["wifi","anom_wifi"],
+    #     "train_dir": ["data/gamutrf/gamutrf-nz-anon-wifi", "data/gamutrf/gamutrf-nz-nonanon-wifi"],
+    #     "val_dir": ["data/gamutrf/gamutrf-nz-wifi"],
+    #     "iq_epochs": 40,
+    #     "spec_epochs": 100,
+    #     "notes": "Wi-Fi vs anomalous Wi-Fi, train on Ettus B200Mini RX/TX, validate on real Wi-Fi AP TX & Ettus B200Mini RX, anarkiwi collect"
+    # },
+    # "experiment_5": {
+    #     "experiment_name": "experiment_5",
+    #     "class_list": ["mini2_video","mini2_telem"],
+    #     "train_dir": ["dev_data/torchsig_train/samples"],
+    #     "val_dir": ["data/gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/test_iq_label"],
+    #     "iq_epochs": 40,
+    #     "spec_epochs": 100,
+    #     "spec_force_yolo_label_larger": True,
+    #     "notes": "DJI Mini2, Ettus B200Mini RX, train on copy of lab collection gamutrf/gamutrf-arl/01_30_23/mini2, validate on field collect gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/test_iq_label"
+    # },
+    # "experiment_6": {
+    #     "experiment_name": "experiment_6",
+    #     "class_list": ["wifi","anom_wifi"],
+    #     "train_dir": [
+    #         "data/gamutrf/wifi-data-03082024/20msps/normal/train",
+    #         "data/gamutrf/wifi-data-03082024/20msps/normal/test",
+    #         "data/gamutrf/wifi-data-03082024/20msps/normal/inference",
+    #         "data/gamutrf/wifi-data-03082024/20msps/mod/train",
+    #         "data/gamutrf/wifi-data-03082024/20msps/mod/test",
+    #         "data/gamutrf/wifi-data-03082024/20msps/mod/inference",
+    #     ],
+    #     "val_dir": [
+    #         "data/gamutrf/wifi-data-03082024/20msps/normal/validate",
+    #         "data/gamutrf/wifi-data-03082024/20msps/mod/validate",
+    #     ], 
+    #     "iq_num_samples": 16*25, 
+    #     "iq_epochs": 10,
+    #     "iq_batch_size": 16,
+    #     "spec_batch_size": 32,
+    #     "spec_epochs": 40,
+    #     "spec_n_fft": 16,
+    #     "spec_time_dim": 25,
+    #     "notes": "Ettus B200Mini RX, emair collect"
+    # },
+    # "experiment_7": {
+    #     "experiment_name": "experiment_7",
+    #     "class_list": ["wifi","anom_wifi"],
+    #     "train_dir": ["data/gamutrf/gamutrf-wifi-and-anom-bladerf"],
+    #     "iq_epochs": 40,
+    #     "spec_epochs": 40,
+    #     "notes": "Wi-Fi vs anomalous Wi-Fi, BladeRF, anarkiwi collect"
+    # },
+    # "experiment_8": {
+    #     "experiment_name": "experiment_8",
+    #     "class_list": ["wifi","anom_wifi"],
+    #     "train_dir": ["data/gamutrf/gamutrf-wifi-and-anom-bladerf"],
+    #     "val_dir": ["data/gamutrf/gamutrf-nz-anon-wifi", "data/gamutrf/gamutrf-nz-nonanon-wifi"],
+    #     "iq_epochs": 40,
+    #     "spec_epochs": 70,
+    #     "spec_skip_export": True, # USE WITH CAUTION (but speeds up large directories significantly): skip after first run if using separate train/val directories 
+    #     "notes": "Wi-Fi vs anomalous Wi-Fi, validate on BladeRF TX & Ettus B200Mini RX, train on Ettus B200Mini RX/TX, anarkiwi collect"
+    # },
+    # "experiment_9": {
+    #     "experiment_name": "experiment_9",
+    #     "class_list": ["mini2_video","mini2_telem"],
+    #     "train_dir": ["dev_data/torchsig_train/samples"],
+    #     "val_dir": ["data/gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/test_iq_label"],
+    #     "iq_epochs": 40,
+    #     "spec_epochs": 200,
+    #     "spec_yolo_augment": True,
+    #     "notes": "DJI Mini2, Ettus B200Mini RX, train on copy of lab collection gamutrf/gamutrf-arl/01_30_23/mini2, validate on field collect gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/test_iq_label"
+    # },
+    # "experiment_10": {
+    #     "experiment_name": "experiment_10",
+    #     "class_list": ["wifi","anom_wifi"],
+    #     "train_dir": ["data/gamutrf/anom_wifi/train_gamutrf-nz-anon-wifi", "data/gamutrf/anom_wifi/train_gamutrf-nz-nonanon-wifi", "data/gamutrf/anom_wifi/train_gamutrf-wifi-and-anom-bladerf"],
+    #     "val_dir": ["data/gamutrf/anom_wifi/val_gamutrf-nz-anon-wifi", "data/gamutrf/anom_wifi/val_gamutrf-nz-nonanon-wifi"],
+    #     "iq_epochs": 40,
+    #     "spec_epochs": 70,
+    #     "spec_skip_export": True, # USE WITH CAUTION (but speeds up large directories significantly): skip after first run if using separate train/val directories 
+    #     "notes": "Wi-Fi vs anomalous Wi-Fi, train on both BladeRF TX & Ettus B200Mini RX and on Ettus B200Mini RX/TX, validate on Ettus B200Mini RX/TX, anarkiwi collect"
+    # },
+    # "experiment_11": {
+    #     "experiment_name": "experiment_11",
+    #     "class_list": ["wifi","anom_wifi"],
+    #     "train_dir": ["data/gamutrf/anom_wifi/train_gamutrf-nz-anon-wifi", "data/gamutrf/anom_wifi/train_gamutrf-nz-nonanon-wifi", "data/gamutrf/anom_wifi/train_gamutrf-wifi-and-anom-bladerf"],
+    #     "val_dir": ["data/gamutrf/anom_wifi/val_gamutrf-wifi-and-anom-bladerf"],
+    #     "iq_epochs": 40,
+    #     "spec_epochs": 70,
+    #     "spec_skip_export": True, # USE WITH CAUTION (but speeds up large directories significantly): skip after first run if using separate train/val directories 
+    #     "notes": "Wi-Fi vs anomalous Wi-Fi, train on both BladeRF TX & Ettus B200Mini RX and on Ettus B200Mini RX/TX, validate on BladeRF TX, anarkiwi collect"
+    # },
 }