Fix broken links in README. Fix broken parameter in annotate.

Author: ltindall

README.md

@@ -100,7 +100,7 @@ annotation_utils.annotate(
                 avg_window_len=256,                 # The number of samples over which to average signal power
                 avg_duration=0.25,                  # The number of seconds, from the start of the recording to use to automatically calculate the SNR threshold, if it is None then all of the samples will be used
                 debug=False,    
-                estimate_frequency=True,            # Whether the frequency bounds for an annotation should be calculated. estimate_frequency needs to be enabled if you use min/max_bandwidth
+                set_bandwidth=10000000,            # Manually set the bandwidth of the signals in Hz, if this parameter is set, then spectral_energy_threshold is ignored
                 spectral_energy_threshold=0.95,     # Percentage used to determine the upper and lower frequency bounds for an annotation
                 force_threshold_db=-58,             # Used to manually set the threshold used for detecting a signal and creating an annotation. If None, then the automatic threshold calculation will be used instead.
                 overwrite=False,                    # If True, any existing annotations in the .sigmf-meta file will be removed
@@ -138,7 +138,7 @@ After you have finished labeling your data, the next step is to train a model on
 
 ### Configure
 
-This repo provides an automated script for training and evaluating models. To do this, configure the [run_experiments.py](./run_experiments.py) file to point to the data you want to use and set the training parameters:
+This repo provides an automated script for training and evaluating models. To do this, configure the [run_experiments.py](rfml/run_experiments.py) file to point to the data you want to use and set the training parameters:
 
 ```python
     "experiment_0": { # A name to refer to the experiment
@@ -186,29 +186,29 @@ This will generate a **.mar** file in the [models/](./models/) folder. [GamutRF]
 ## Files
 
 
-[annotation_utils.py](annotation_utils.py) - DSP based automated labelling tools
+[annotation_utils.py](rfml/annotation_utils.py) - DSP based automated labelling tools
 
-[auto_label.py](auto_label.py) - CV based automated labelling tools
+[auto_label.py](rfml/auto_label.py) - CV based automated labelling tools
 
-[data.py](data.py) - RF data operations tool
+[data.py](rfml/data.py) - RF data operations tool
 
-[experiment.py](experiment.py) - Class to manage experiments 
+[experiment.py](rfml/experiment.py) - Class to manage experiments 
 
-[models.py](models.py) - Class for I/Q models (based on TorchSig) 
+[models.py](rfml/models.py) - Class for I/Q models (based on TorchSig) 
 
-[run_experiments.py](run_experiments.py) - Experiment configurations and run script
+[run_experiments.py](rfml/run_experiments.py) - Experiment configurations and run script
 
-[sigmf_pytorch_dataset.py](sigmf_pytorch_dataset.py) - PyTorch style dataset class for SigMF data (based on TorchSig) 
+[sigmf_pytorch_dataset.py](rfml/sigmf_pytorch_dataset.py) - PyTorch style dataset class for SigMF data (based on TorchSig) 
 
-[spectrogram.py](spectrogram.py) - Spectrogram tools 
+[spectrogram.py](rfml/spectrogram.py) - Spectrogram tools 
 
-[test_data.py](test_data.py) - Test for data.py (might be outdated)
+[test_data.py](rfml/test_data.py) - Test for data.py (might be outdated)
 
-[train_iq.py](train_iq.py) - Training script for I/Q models
+[train_iq.py](rfml/train_iq.py) - Training script for I/Q models
 
-[train_spec.py](train_spec.py) - Training script for spectrogram models
+[train_spec.py](rfml/train_spec.py) - Training script for spectrogram models
 
-[zst_parse.py](zst_parse.py) - ZST file parsing tool, for GamutRF-style filenames  
+[zst_parse.py](rfml/zst_parse.py) - ZST file parsing tool, for GamutRF-style filenames  
 
 The [notebooks/](./notebooks/) directory contains various experiments we have conducted during development.
 

label_scripts/label_env.py

@@ -24,7 +24,7 @@
 for label in data_globs:
     for data_glob in data_globs[label]:
         for f in tqdm(glob.glob(str(Path(data_glob)))):
-            # annotation_utils.annotate(f, label=label, avg_window_len=256, avg_duration=0.25, debug=True, estimate_frequency=True, spectral_energy_threshold=0.99, force_threshold_db=-40)
+            # annotation_utils.annotate(f, label=label, avg_window_len=256, avg_duration=0.25, debug=True, spectral_energy_threshold=0.99, force_threshold_db=-40)
 
             data_obj = data_class.Data(f)
             annotation_utils.reset_annotations(data_obj)
@@ -34,7 +34,6 @@
                 avg_window_len=256,
                 avg_duration=0.10,
                 debug=False,
-                estimate_frequency=True,
                 spectral_energy_threshold=0.90,
                 # force_threshold_db=-48,
                 overwrite=True,

label_scripts/label_mavic.py

@@ -23,7 +23,7 @@
 for label in data_globs:
     for data_glob in data_globs[label]:
         for f in tqdm(glob.glob(str(Path(data_glob)))):
-            # annotation_utils.annotate(f, label=label, avg_window_len=256, avg_duration=0.25, debug=True, estimate_frequency=True, spectral_energy_threshold=0.99, force_threshold_db=-40)
+            # annotation_utils.annotate(f, label=label, avg_window_len=256, avg_duration=0.25, debug=True, spectral_energy_threshold=0.99, force_threshold_db=-40)
 
             print(f"\nAnnotating {f}\n-----------------------------------\n")
 
@@ -35,7 +35,6 @@
             #     avg_window_len=256,
             #     avg_duration=0.10,
             #     debug=False,
-            #     estimate_frequency=True,
             #     spectral_energy_threshold=0.90,
             #     #force_threshold_db=-48,
             #     overwrite=True,
@@ -51,7 +50,6 @@
                 avg_window_len=256,
                 avg_duration=0.25,
                 debug=False,
-                estimate_frequency=True,
                 spectral_energy_threshold=0.99,
                 force_threshold_db=-58,
                 overwrite=True,

label_scripts/label_mini2.py

@@ -21,7 +21,7 @@
 for label in data_globs:
     for data_glob in data_globs[label]:
         for f in tqdm(glob.glob(str(Path(data_glob)))):
-            # annotation_utils.annotate(f, label=label, avg_window_len=256, avg_duration=0.25, debug=True, estimate_frequency=True, spectral_energy_threshold=0.99, force_threshold_db=-40)
+            # annotation_utils.annotate(f, label=label, avg_window_len=256, avg_duration=0.25, debug=True, spectral_energy_threshold=0.99, force_threshold_db=-40)
 
             data_obj = data_class.Data(f)
             annotation_utils.reset_annotations(data_obj)
@@ -31,7 +31,6 @@
                 avg_window_len=256,
                 avg_duration=0.25,
                 debug=False,
-                estimate_frequency=True,
                 spectral_energy_threshold=0.95,
                 force_threshold_db=-58,
                 overwrite=False,
@@ -46,7 +45,6 @@
                 avg_window_len=256,
                 avg_duration=0.25,
                 debug=False,
-                estimate_frequency=True,
                 spectral_energy_threshold=0.95,
                 force_threshold_db=-58,
                 overwrite=False,

label_scripts/label_nz_wifi_multiprocessing.py

@@ -73,7 +73,7 @@ def try_fn():
                     "filename": f,
                     "label": label[0],
                     "avg_duration": 3,
-                    "estimate_frequency": label[1],
+                    "set_bandwidth": label[1],
                     "time_start_stop": 5,
                 }
             )
@@ -89,11 +89,7 @@ def try_fn():
 resource.setrlimit(resource.RLIMIT_AS, (memory_limit, memory_limit))
 
 pool = Pool(16)
-# common_args = {
-#     'avg_duration':3,
-#     'estimate_frequency':
-#     'time_start_stop': 5,
-# }
+
 # pool.starmap(partial(annotation_utils.annotate, **common_args), job_args)
 
 

notebooks/Label_DJI.ipynb

@@ -370,12 +370,12 @@
     "for label in data_globs:\n",
     "    for data_glob in data_globs[label]:\n",
     "        for f in tqdm(glob.glob(str(Path(data_glob)))):\n",
-    "            # annotation_utils.annotate(f, label=label, avg_window_len=256, avg_duration=0.25, debug=True, estimate_frequency=True, spectral_energy_threshold=0.99, force_threshold_db=-40)\n",
+    "            # annotation_utils.annotate(f, label=label, avg_window_len=256, avg_duration=0.25, debug=True, spectral_energy_threshold=0.99, force_threshold_db=-40)\n",
     "            print(f\"{f=}\")\n",
     "            data_obj = data_class.Data(f)\n",
     "            annotation_utils.reset_annotations(data_obj)\n",
-    "            annotation_utils.annotate(f, label=\"mini2_video\", avg_window_len=256, avg_duration=0.25, debug=True, estimate_frequency=True, spectral_energy_threshold=0.95, force_threshold_db=-58, overwrite=False, min_bandwidth=16e6, min_annotation_length=10000, max_annotations=10, dc_block=True, verbose=False, dry_run=True)\n",
-    "            annotation_utils.annotate(f, label=\"mini2_telem\", avg_window_len=256, avg_duration=0.25, debug=True, estimate_frequency=True, spectral_energy_threshold=0.95, force_threshold_db=-58, overwrite=False, max_bandwidth=16e6, min_annotation_length=10000, max_annotations=10, dc_block=True, verbose=False, dry_run=True)\n",
+    "            annotation_utils.annotate(f, label=\"mini2_video\", avg_window_len=256, avg_duration=0.25, debug=True, spectral_energy_threshold=0.95, force_threshold_db=-58, overwrite=False, min_bandwidth=16e6, min_annotation_length=10000, max_annotations=10, dc_block=True, verbose=False, dry_run=True)\n",
+    "            annotation_utils.annotate(f, label=\"mini2_telem\", avg_window_len=256, avg_duration=0.25, debug=True, spectral_energy_threshold=0.95, force_threshold_db=-58, overwrite=False, max_bandwidth=16e6, min_annotation_length=10000, max_annotations=10, dc_block=True, verbose=False, dry_run=True)\n",
     "            "
    ]
   },

rfml/annotation_utils.py

@@ -48,7 +48,7 @@ def annotate_power_squelch(
     avg_window_len,
     label=None,
     skip_validate=False,
-    estimate_frequency=False,
+    spectral_energy_threshold=False,
     dry_run=False,
     min_annotation_length=400,
     min_bandwidth=None,
@@ -59,6 +59,7 @@ def annotate_power_squelch(
     verbose=False,
     n_seek_samples=None,
     n_samples=None,
+    set_bandwidth=None,
 ):
     iq_samples = data_obj.get_samples(
         n_seek_samples=n_seek_samples, n_samples=n_samples
@@ -73,17 +74,23 @@ def annotate_power_squelch(
         if min_annotation_length and (stop - start < min_annotation_length):
             continue
 
-        if isinstance(estimate_frequency, bool):
-            if estimate_frequency:
-                estimate_frequency = 0.94
-            else:
-                estimate_frequency = None
-        if isinstance(estimate_frequency, float):
+        if isinstance(spectral_energy_threshold, bool) and spectral_energy_threshold:
+            spectral_energy_threshold = 0.94
+
+        if set_bandwidth:
+            freq_lower_edge = (
+                data_obj.metadata["captures"][0]["core:frequency"] - set_bandwidth / 2
+            )
+            freq_upper_edge = (
+                data_obj.metadata["captures"][0]["core:frequency"] + set_bandwidth / 2
+            )
+
+        elif isinstance(spectral_energy_threshold, float):
             freq_lower_edge, freq_upper_edge = get_occupied_bandwidth(
                 iq_samples[start:stop],
                 data_obj.metadata["global"]["core:sample_rate"],
                 data_obj.metadata["captures"][0]["core:frequency"],
-                spectral_energy_threshold=estimate_frequency,
+                spectral_energy_threshold=spectral_energy_threshold,
                 dc_block=dc_block,
                 verbose=verbose,
             )
@@ -141,7 +148,7 @@ def annotate(
     debug=False,
     dry_run=False,
     min_annotation_length=400,
-    estimate_frequency=True,
+    spectral_energy_threshold=True,
     force_threshold_db=None,
     overwrite=True,
     min_bandwidth=None,
@@ -150,6 +157,7 @@ def annotate(
     dc_block=None,
     verbose=False,
     time_start_stop=None,
+    set_bandwidth=None,
 ):
 
     data_obj = data_class.Data(filename)
@@ -249,7 +257,7 @@ def median_absolute_deviation(series):
         avg_window_len,
         label=label,
         skip_validate=True,
-        estimate_frequency=estimate_frequency,
+        spectral_energy_threshold=spectral_energy_threshold,
         min_bandwidth=min_bandwidth,
         max_bandwidth=max_bandwidth,
         dry_run=dry_run,
@@ -260,6 +268,7 @@ def median_absolute_deviation(series):
         verbose=verbose,
         n_seek_samples=n_seek_samples,
         n_samples=n_samples,
+        set_bandwidth=set_bandwidth,
     )