Skip to content

Commit 12b265b

Browse files
ltindallltindall
committedSep 13, 2024
Refactor annotation_utils and other changes.
·
v0.1.0
1 parent 063d842 commit 12b265b

23 files changed

+3946
-1202
lines changed
 

‎README.md

+35-26
Original file line numberDiff line numberDiff line change
@@ -95,21 +95,31 @@ In the labeling scripts, the settings for autolabeling need to be tuned for the
9595

9696
```python
9797
annotation_utils.annotate(
98-
f,
99-
label="mavic3_video", # This is the label that is applied to all of the matching annotations
100-
avg_window_len=256, # The number of samples over which to average signal power
101-
avg_duration=0.25, # The number of seconds, from the start of the recording to use to automatically calculate the SNR threshold, if it is None then all of the samples will be used
102-
debug=False,
103-
set_bandwidth=10000000, # Manually set the bandwidth of the signals in Hz, if this parameter is set, then spectral_energy_threshold is ignored
104-
spectral_energy_threshold=0.95, # Percentage used to determine the upper and lower frequency bounds for an annotation
105-
force_threshold_db=-58, # Used to manually set the threshold used for detecting a signal and creating an annotation. If None, then the automatic threshold calculation will be used instead.
106-
overwrite=False, # If True, any existing annotations in the .sigmf-meta file will be removed
107-
min_bandwidth=16e6, # The minimum bandwidth (in Hz) of a signal to annotate
108-
max_bandwidth=None, # The maximum bandwidth (in Hz) of a signal to annotate
109-
min_annotation_length=10000, # The minimum numbers of samples in length a signal needs to be in order for it to be annotated. This is directly related to the sample rate a signal was captured at and does not take into account bandwidth. So 10000 samples at 20,000,000 samples per second, would mean a minimum transmission length of 0.0005 seconds
110-
# max_annotations=500, # The maximum number of annotations to automatically add
111-
dc_block=True # De-emphasize the DC spike when trying to calculate the frequencies for a signal
112-
)
98+
rfml.data.Data(filename),
99+
avg_window_len=256, # The window size to use when averaging signal power
100+
power_estimate_duration=0.1, # Process the file in chunks of power_estimate_duration seconds
101+
debug_duration=0.25, # If debug==True, then plot debug_duration seconds of data in debug plots
102+
debug=False, # Set True to enable debugging plots
103+
verbose=False, # Set True to eanble verbose messages
104+
dry_run=False, # Set True to disable annotations being written to SigMF-Meta file.
105+
bandwidth_estimation=True, # If set to True, will estimate signal bandwidth using Gaussian Mixture Models. If set to a float will estimate signal bandwidth using spectral thresholding.
106+
force_threshold_db=None, # Used to manually set the threshold used for detecting a signal and creating an annotation. If None, then the automatic threshold calculation will be used instead.
107+
overwrite=True, # If True, any existing annotations in the .sigmf-meta file will be removed
108+
max_annotations=None, # If set, limits the number of annotations to add.
109+
dc_block=None, # De-emphasize the DC spike when trying to calculate the frequencies for a signal
110+
time_start_stop=None, # Sets the start/stop time for annotating the recording (must be tuple or list of length 2).
111+
n_components = None, # Sets the number of mixture components to use when calculating signal detection threshold. If not set, then automatically calculated from labels.
112+
n_init=1, # Number of initializations to use in Gaussian Mixture Method. Increasing this number can significantly increase run time.
113+
fft_len=256, # FFT length used in calculating bandwidth
114+
labels = { # The labels dictionary defines the annotations that the script will attempt to find.
115+
"mavic3_video": { # The dictionary keys define the annotation labels. Only a key is necessary.
116+
"bandwidth_limits": (8e6, None), # Optional. Set min/max bandwidth limit for a signal. If None, no min/max limit.
117+
"annotation_length": (10000, None), # Optional. Set min/max annoation length in number of samples. If None, no min/max limit.
118+
"annotation_seconds": (0.0001, 0.0025), # Optional. Set min/max annotation length in seconds. If None, no min/max limit.
119+
"set_bandwidth": (-8.5e6, 9.5e6) # Optional. Ignore bandwidth estimation, set bandwidth manually. Limits are in relation to center frequency.
120+
}
121+
}
122+
)
113123
```
114124

115125
### Tips for Tuning Autolabeling
@@ -138,7 +148,7 @@ After you have finished labeling your data, the next step is to train a model on
138148

139149
### Configure
140150

141-
This repo provides an automated script for training and evaluating models. To do this, configure the [run_experiments.py](rfml/run_experiments.py) file to point to the data you want to use and set the training parameters:
151+
This repo provides an automated script for training and evaluating models. To do this, configure the [mixed_experiments.py](rfml/mixed_experiments.py) file or create your own to point to the data you want to use and set the training parameters:
142152

143153
```python
144154
"experiment_0": { # A name to refer to the experiment
@@ -150,10 +160,10 @@ This repo provides an automated script for training and evaluating models. To do
150160
}
151161
```
152162

153-
Once you have the **run_experiments.py** file configured, run it:
163+
Once you have the **mixed_experiments.py** file configured, run it:
154164

155165
```bash
156-
python3 run_experiments.py
166+
python3 mixed_experiments.py
157167
```
158168

159169
Once the training has completed, it will print out the logs location, model accuracy, and the location of the best checkpoint:
@@ -170,18 +180,15 @@ Best Model Checkpoint: lightning_logs/version_5/checkpoints/experiment_logs/expe
170180

171181
### Convert & Export IQ Models
172182

173-
Once you have a trained model, you need to convert it into a portable format that can easily be served by TorchServe. To do this, use **convert_model.py**:
183+
Once you have a trained model, you need to convert it into a portable format that can easily be served by TorchServe. To do this, use **export_model.py**:
174184

175185
```bash
176-
python3 convert_model.py --model_name=drone_detect --checkpoint=lightning_logs/version_5/checkpoints/experiment_logs/experiment_1/iq_checkpoints/checkpoint.ckpt
186+
python3 rfml/export_model.py --model_name=drone_detect --checkpoint=lightning_logs/version_5/checkpoints/experiment_logs/experiment_1/iq_checkpoints/checkpoint.ckpt
177187
```
178-
This will export a **_torchscript.pt** file.
188+
This will create a **_torchscript.pt** and **_torchserve.pt** file in the weights folder.
179189

180-
```bash
181-
torch-model-archiver --force --model-name drone_detect --version 1.0 --serialized-file weights/drone_detect_torchscript.pt --handler custom_handlers/iq_custom_handler.py --export-path models/ -r custom_handler/requirements.txt
182-
```
190+
A **.mar** file will also be created in the [models/](./models/) folder. [GamutRF](https://github.com/IQTLabs/gamutRF) can run this model and use it to classify signals.
183191

184-
This will generate a **.mar** file in the [models/](./models/) folder. [GamutRF](https://github.com/IQTLabs/gamutRF) can run this model and use it to classify signals.
185192

186193
## Files
187194

@@ -194,9 +201,11 @@ This will generate a **.mar** file in the [models/](./models/) folder. [GamutRF]
194201

195202
[experiment.py](rfml/experiment.py) - Class to manage experiments
196203

204+
[export_model.py](rfml/export_model.py) - Convert and export model checkpoints to Torchscript/Torchserve/MAR format.
205+
197206
[models.py](rfml/models.py) - Class for I/Q models (based on TorchSig)
198207

199-
[run_experiments.py](rfml/run_experiments.py) - Experiment configurations and run script
208+
[experiments/](experiments/) - Experiment configurations and run script
200209

201210
[sigmf_pytorch_dataset.py](rfml/sigmf_pytorch_dataset.py) - PyTorch style dataset class for SigMF data (based on TorchSig)
202211

‎experiments/dji_mini2_wifi_experiments.py

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
from rfml.experiment import *
2+
3+
# Ensure that data directories have sigmf-meta files with annotations
4+
# Annotations can be generated using scripts in label_scripts directory or notebooks/Label_WiFi.ipynb and notebooks/Label_DJI.ipynb
5+
6+
spec_epochs = 0
7+
iq_epochs = 10
8+
iq_only_start_of_burst = False
9+
iq_num_samples = 4000
10+
iq_early_stop = 3
11+
iq_train_limit = 0.01
12+
iq_val_limit = 0.1
13+
14+
experiments = {
15+
"experiment_nz_wifi_arl_mini2_pdx_mini2_to_leesburg_mini2": {
16+
"class_list": ["mini2_video", "mini2_telem", "wifi"],
17+
"train_dir": [
18+
"/data/s3_gamutrf/gamutrf-nz-wifi",
19+
"/data/s3_gamutrf/gamutrf-arl/01_30_23/mini2",
20+
"/data/s3_gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/iq_recordings",
21+
],
22+
"val_dir": [
23+
"/data/s3_gamutrf/gamutrf-nz-wifi",
24+
"/data/s3_gamutrf/gamutrf-birdseye-field-days/leesburg_field_day_2022_06_15/iq_recordings",
25+
],
26+
"iq_epochs": iq_epochs,
27+
"spec_epochs": spec_epochs,
28+
"iq_only_start_of_burst": iq_only_start_of_burst,
29+
"iq_early_stop": iq_early_stop,
30+
"iq_train_limit": iq_train_limit,
31+
"iq_val_limit": iq_val_limit,
32+
"notes": "",
33+
},
34+
"experiment_nz_wifi_arl_mini2_to_leesburg_mini2": {
35+
"class_list": ["mini2_video", "mini2_telem", "wifi"],
36+
"train_dir": [
37+
"/data/s3_gamutrf/gamutrf-nz-wifi",
38+
"/data/s3_gamutrf/gamutrf-arl/01_30_23/mini2",
39+
],
40+
"val_dir": [
41+
"/data/s3_gamutrf/gamutrf-nz-wifi",
42+
"/data/s3_gamutrf/gamutrf-birdseye-field-days/leesburg_field_day_2022_06_15/iq_recordings",
43+
],
44+
"iq_epochs": iq_epochs,
45+
"spec_epochs": spec_epochs,
46+
"iq_only_start_of_burst": iq_only_start_of_burst,
47+
"iq_early_stop": iq_early_stop,
48+
"iq_train_limit": iq_train_limit,
49+
"iq_val_limit": iq_val_limit,
50+
"notes": "",
51+
},
52+
"experiment_nz_wifi_pdx_mini2_to_leesburg_mini2": {
53+
"class_list": ["mini2_video", "mini2_telem", "wifi"],
54+
"train_dir": [
55+
"/data/s3_gamutrf/gamutrf-nz-wifi",
56+
"/data/s3_gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/iq_recordings",
57+
],
58+
"val_dir": [
59+
"/data/s3_gamutrf/gamutrf-nz-wifi",
60+
"/data/s3_gamutrf/gamutrf-birdseye-field-days/leesburg_field_day_2022_06_15/iq_recordings",
61+
],
62+
"iq_epochs": iq_epochs,
63+
"spec_epochs": spec_epochs,
64+
"iq_only_start_of_burst": iq_only_start_of_burst,
65+
"iq_early_stop": iq_early_stop,
66+
"iq_train_limit": iq_train_limit,
67+
"iq_val_limit": iq_val_limit,
68+
"notes": "",
69+
},
70+
"experiment_nz_wifi_arl_mini2_to_pdx_mini2": {
71+
"class_list": ["mini2_video", "mini2_telem", "wifi"],
72+
"train_dir": [
73+
"/data/s3_gamutrf/gamutrf-nz-wifi",
74+
"/data/s3_gamutrf/gamutrf-arl/01_30_23/mini2",
75+
],
76+
"val_dir": [
77+
"/data/s3_gamutrf/gamutrf-nz-wifi",
78+
"/data/s3_gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/iq_recordings",
79+
],
80+
"iq_epochs": iq_epochs,
81+
"spec_epochs": spec_epochs,
82+
"iq_only_start_of_burst": iq_only_start_of_burst,
83+
"iq_early_stop": iq_early_stop,
84+
"iq_train_limit": iq_train_limit,
85+
"iq_val_limit": iq_val_limit,
86+
"notes": "",
87+
},
88+
"experiment_nz_wifi_leesburg_mini2_to_pdx_mini2": {
89+
"class_list": ["mini2_video", "mini2_telem", "wifi"],
90+
"train_dir": [
91+
"/data/s3_gamutrf/gamutrf-nz-wifi",
92+
"/data/s3_gamutrf/gamutrf-birdseye-field-days/leesburg_field_day_2022_06_15/iq_recordings",
93+
],
94+
"val_dir": [
95+
"/data/s3_gamutrf/gamutrf-nz-wifi",
96+
"/data/s3_gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/iq_recordings",
97+
],
98+
"iq_epochs": iq_epochs,
99+
"spec_epochs": spec_epochs,
100+
"iq_only_start_of_burst": iq_only_start_of_burst,
101+
"iq_early_stop": iq_early_stop,
102+
"iq_train_limit": iq_train_limit,
103+
"iq_val_limit": iq_val_limit,
104+
"notes": "",
105+
},
106+
"experiment_nz_wifi_leesburg_mini2_pdx_mini2_to_arl_mini2": {
107+
"class_list": ["mini2_video", "mini2_telem", "wifi"],
108+
"train_dir": [
109+
"/data/s3_gamutrf/gamutrf-nz-wifi",
110+
"/data/s3_gamutrf/gamutrf-birdseye-field-days/leesburg_field_day_2022_06_15/iq_recordings",
111+
"/data/s3_gamutrf/gamutrf-birdseye-field-days/pdx_field_day_2022_05_26/iq_recordings",
112+
],
113+
"val_dir": [
114+
"/data/s3_gamutrf/gamutrf-nz-wifi",
115+
"/data/s3_gamutrf/gamutrf-arl/01_30_23/mini2",
116+
],
117+
"iq_epochs": iq_epochs,
118+
"spec_epochs": spec_epochs,
119+
"iq_only_start_of_burst": iq_only_start_of_burst,
120+
"iq_early_stop": iq_early_stop,
121+
"iq_train_limit": iq_train_limit,
122+
"iq_val_limit": iq_val_limit,
123+
"notes": "",
124+
},
125+
}
126+
127+
128+
if __name__ == "__main__":
129+
130+
experiments_to_run = [
131+
# "experiment_nz_wifi_arl_mini2_pdx_mini2_to_leesburg_mini2",
132+
# "experiment_nz_wifi_arl_mini2_to_leesburg_mini2",
133+
# "experiment_nz_wifi_pdx_mini2_to_leesburg_mini2",
134+
# "experiment_nz_wifi_arl_mini2_to_pdx_mini2",
135+
# "experiment_nz_wifi_leesburg_mini2_to_pdx_mini2",
136+
"experiment_nz_wifi_leesburg_mini2_pdx_mini2_to_arl_mini2"
137+
]
138+
139+
train({name: experiments[name] for name in experiments_to_run})

‎rfml/run_experiments.py ‎experiments/mixed_experiments.py

+1-49
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,4 @@
1-
from pathlib import Path
2-
31
from rfml.experiment import *
4-
from rfml.train_iq import *
5-
from rfml.train_spec import *
6-
72

83
# Ensure that data directories have sigmf-meta files with annotations
94
# Annotations can be generated using scripts in label_scripts directory or notebooks/Label_WiFi.ipynb and notebooks/Label_DJI.ipynb
@@ -262,47 +257,4 @@
262257
# "experiment_mavic3",
263258
]
264259

265-
for experiment_name in experiments_to_run:
266-
print(f"Running {experiment_name}")
267-
try:
268-
exp = Experiment(
269-
experiment_name=experiment_name, **experiments[experiment_name]
270-
)
271-
272-
logs_timestamp = datetime.now().strftime("%m_%d_%Y_%H_%M_%S")
273-
274-
if exp.iq_epochs > 0:
275-
train_iq(
276-
train_dataset_path=exp.train_dir,
277-
val_dataset_path=exp.val_dir,
278-
num_iq_samples=exp.iq_num_samples,
279-
only_use_start_of_burst=exp.iq_only_start_of_burst,
280-
epochs=exp.iq_epochs,
281-
batch_size=exp.iq_batch_size,
282-
class_list=exp.class_list,
283-
output_dir=Path("experiment_logs", exp.experiment_name),
284-
logs_dir=Path("iq_logs", logs_timestamp),
285-
)
286-
else:
287-
print("Skipping IQ training")
288-
289-
if exp.spec_epochs > 0:
290-
train_spec(
291-
train_dataset_path=exp.train_dir,
292-
val_dataset_path=exp.val_dir,
293-
n_fft=exp.spec_n_fft,
294-
time_dim=exp.spec_time_dim,
295-
epochs=exp.spec_epochs,
296-
batch_size=exp.spec_batch_size,
297-
class_list=exp.class_list,
298-
yolo_augment=exp.spec_yolo_augment,
299-
skip_export=exp.spec_skip_export,
300-
force_yolo_label_larger=exp.spec_force_yolo_label_larger,
301-
output_dir=Path("experiment_logs", exp.experiment_name),
302-
logs_dir=Path("spec_logs", logs_timestamp),
303-
)
304-
else:
305-
print("Skipping spectrogram training")
306-
307-
except Exception as error:
308-
print(f"Error: {error}")
260+
train({name: experiments[name] for name in experiments_to_run})

‎experiments/siggen_experiments.py

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import torch
2+
3+
torch.set_float32_matmul_precision("medium")
4+
from rfml.experiment import *
5+
6+
#
7+
# python rfml/siggen_experiments.py
8+
# python convert_model.py --model_name siggen_model --checkpoint /home/ltindall/iqt/rfml/lightning_logs/siggen_experiment/checkpoints/experiment_logs/siggen_experiment/iq_checkpoints/checkpoint-v3.ckpt
9+
# torch-model-archiver --force --model-name siggen_model --version 1.0 --serialized-file rfml/weights/siggen_model_torchscript.pt --handler custom_handlers/iq_custom_handler.py --export-path models/ -r custom_handlers/requirements.txt
10+
# cp models/siggen_model.mar ~/iqt/gamutrf-deploy/docker_rundir/model_store/
11+
# sudo chmod -R 777 /home/ltindall/iqt/gamutrf-deploy/docker_rundir/
12+
#
13+
14+
15+
experiments = {
16+
"siggen_experiment": {
17+
"class_list": ["am", "fm"],
18+
"train_dir": [
19+
"/data/siggen/fm.sigmf-meta",
20+
"/data/siggen/am.sigmf-meta",
21+
],
22+
"val_dir": [
23+
"/data/siggen/fm.sigmf-meta",
24+
"/data/siggen/am.sigmf-meta",
25+
],
26+
"iq_epochs": 10,
27+
"iq_train_limit": 0.5,
28+
"iq_only_start_of_burst": False,
29+
"iq_num_samples": 1024,
30+
"spec_epochs": 0,
31+
}
32+
}
33+
34+
35+
if __name__ == "__main__":
36+
37+
train(experiments)

0 commit comments

Comments
 (0)
Please sign in to comment.