Skip to content

Commit 7473180

Browse files
quic-hanwxionquic-hanwxion
committed
Added more acceptance test coverage and minor fixes to existing tests; Updates to CI/CD pipeline; Added 3 new TF models and updated Torch MobileBert
- Minor clean-up of code to minimize errors from static analysis (pylint) - Added more acceptance tests for a few pytorch segmentation models and applied minor fixes to previous acceptance tests - Config file and documentation updates for a pytorch MobileBert models - Releasing next package release of aimet model zoo (installable wheel file binaries) - Added code, documentation and artifacts the following new Tensorflow models: MobileNet-EdgeTPU, DeepLabV3Plus_Xception_TF2 and DeepLabV3Plus_MBNV2_TF2 Signed-off-by: Hanwen Xiong <[email protected]>
1 parent 7e1bea4 commit 7473180

File tree

98 files changed

+3367
-218
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

98 files changed

+3367
-218
lines changed

‎.pylintrc

+1-1
Original file line numberDiff line numberDiff line change
@@ -558,4 +558,4 @@ valid-metaclass-classmethod-first-arg=mcs
558558

559559
# Exceptions that will emit a warning when being caught. Defaults to
560560
# "Exception"
561-
overgeneral-exceptions=Exception
561+
overgeneral-exceptions=base.Exception

‎AcceptanceTests/CMakeLists.txt

+10-8
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,18 @@ if (ENABLE_TORCH)
1515

1616
endif (ENABLE_TORCH)
1717

18-
#if (ENABLE_TENSORFLOW)
19-
# add_dependencies(AcceptanceTests
20-
# AcceptanceTests.Tensorflow)
21-
#
22-
#endif (ENABLE_TENSORFLOW)
18+
if (ENABLE_TENSORFLOW)
19+
add_dependencies(AcceptanceTests
20+
AcceptanceTests.Tensorflow)
21+
22+
endif (ENABLE_TENSORFLOW)
2323

2424
if (ENABLE_TORCH)
25+
message(STATUS "Torch has been enabled")
2526
add_subdirectory(torch)
2627
endif (ENABLE_TORCH)
2728

28-
#if (ENABLE_TENSORFLOW)
29-
# add_subdirectory(tensorflow)
30-
#endif (ENABLE_TENSORFLOW)
29+
if (ENABLE_TENSORFLOW)
30+
message(STATUS "Tensorflow has been enabled")
31+
add_subdirectory(tensorflow)
32+
endif (ENABLE_TENSORFLOW)

‎AcceptanceTests/tensorflow/CMakeLists.txt

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# =============================================================================
2+
# @@-COPYRIGHT-START-@@
3+
#
4+
# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
5+
#
6+
# @@-COPYRIGHT-END-@@
7+
# =============================================================================
8+
if (ENABLE_CUDA)
9+
set(CUDA_FLAG -m "not blah")
10+
set(USE_CUDA True)
11+
else (ENABLE_CUDA)
12+
set(CUDA_FLAG -m "not cuda")
13+
set(USE_CUDA False)
14+
endif (ENABLE_CUDA)
15+
16+
add_custom_target( AcceptanceTests.Tensorflow )
17+
18+
file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/*quanteval.py")
19+
foreach(filename ${files})
20+
get_filename_component( testname "${filename}" NAME_WE )
21+
string(SUBSTRING ${testname} 5 -1 formatted_name)
22+
message(STATUS "Now running Tensorflow Test: ${formatted_name}")
23+
24+
add_custom_target(AcceptanceTests.Tensorflow.${testname}
25+
VERBATIM COMMAND . /opt/${formatted_name}-venv/bin/activate && export HF_HOME=/opt/cache &&
26+
${CMAKE_COMMAND} -E env
27+
${MZ_PYTHONPATH}:$$PYTHONPATH
28+
python -m pytest -s ${filename}
29+
${CUDA_FLAG} --junitxml=${CMAKE_CURRENT_BINARY_DIR}/py_test_output_${testname}.xml)
30+
31+
add_dependencies( AcceptanceTests.Tensorflow AcceptanceTests.Tensorflow.${testname} )
32+
endforeach( filename )

‎AcceptanceTests/tensorflow/conftest.py

100644100755
+16
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,20 @@ def tiny_mscoco_tfrecords(test_data_path):
5252

5353
yield tiny_mscoco_tfrecords
5454

55+
@pytest.fixture(scope='module')
56+
def PascalVOC_segmentation_test_data_path(test_data_path):
57+
if test_data_path is not None:
58+
pascalVOC_segmentation_path = (test_data_path / "model_zoo_datasets/tiny_VOCdevkit").as_posix()
59+
else:
60+
pascalVOC_segmentation_path = None
61+
62+
yield pascalVOC_segmentation_path
63+
64+
#@pytest.fixture(scope='module')
65+
#def tiny_imageNet_tfrecords(test_data_path):
66+
# if test_data_path is not None:
67+
# tiny_imageNet_tfrecords = (test_data_path / "model_zoo_datasets/tiny_imageNet_tfrecords").as_posix()
68+
# else:
69+
# tiny_imageNet_tfrecords = None
5570

71+
# yield tiny_imageNet_tfrecords

‎AcceptanceTests/tensorflow/staging/test_deeplabv3plus_mbnv2_tf2_quanteval.py

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# /usr/bin/env python3
2+
# -*- mode: python -*-
3+
# =============================================================================
4+
# @@-COPYRIGHT-START-@@
5+
#
6+
# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
7+
# Changes from QuIC are licensed under the terms and conditions at
8+
# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
9+
#
10+
# @@-COPYRIGHT-END-@@
11+
# =============================================================================
12+
13+
""" acceptance test for deeplabv3plus mbnv2"""
14+
15+
import pytest
16+
from aimet_zoo_tensorflow.deeplabv3plus_tf2.evaluators import deeplabv3plus_tf2_quanteval
17+
18+
@pytest.mark.slow
19+
@pytest.mark.cuda
20+
@pytest.mark.sementic_segmentation
21+
# pylint:disable = redefined-outer-name
22+
@pytest.mark.parametrize("model_config", ["deeplabv3plus_mbnv2_w8a8"])
23+
def test_quanteval_deeplabv3plus_mbnv2_tf2(model_config, PascalVOC_segmentation_test_data_path):
24+
"""mobiledet edgetpu image classification test"""
25+
26+
if PascalVOC_segmentation_test_data_path is None:
27+
pytest.xfail(f'Dataset path is not set')
28+
29+
deeplabv3plus_tf2_quanteval.main(
30+
[
31+
"--model-config",
32+
model_config,
33+
"--dataset-path",
34+
PascalVOC_segmentation_test_data_path
35+
]
36+
)
37+

‎AcceptanceTests/tensorflow/staging/test_deeplabv3plus_xception_tf2_quanteval.py

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/usr/bin/env python3
2+
# -*- mode: python -*-
3+
# =============================================================================
4+
# @@-COPYRIGHT-START-@@
5+
#
6+
# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
7+
#
8+
# @@-COPYRIGHT-END-@@
9+
# =============================================================================
10+
11+
""" acceptance test for deeplabv3plus xception"""
12+
13+
import pytest
14+
from aimet_zoo_tensorflow.deeplabv3plus_tf2.evaluators import deeplabv3plus_tf2_quanteval
15+
16+
@pytest.mark.slow
17+
@pytest.mark.cuda
18+
@pytest.mark.sementic_segmentation
19+
# pylint:disable = redefined-outer-name
20+
@pytest.mark.parametrize("model_config", ["deeplabv3plus_xception_w8a8"])
21+
def test_quanteval_deeplabv3plus_xception_tf2(model_config, PascalVOC_segmentation_test_data_path):
22+
"""mobiledet edgetpu image classification test"""
23+
24+
if PascalVOC_segmentation_test_data_path is None:
25+
pytest.xfail(f'Dataset path is not set')
26+
27+
deeplabv3plus_tf2_quanteval.main(
28+
[
29+
"--model-config",
30+
model_config,
31+
"--dataset-path",
32+
PascalVOC_segmentation_test_data_path
33+
]
34+
)
35+

‎AcceptanceTests/tensorflow/staging/test_mobilenet_edgetpu_quanteval.py

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# /usr/bin/env python3
2+
# -*- mode: python -*-
3+
# =============================================================================
4+
# @@-COPYRIGHT-START-@@
5+
#
6+
# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
7+
# Changes from QuIC are licensed under the terms and conditions at
8+
# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
9+
#
10+
# @@-COPYRIGHT-END-@@
11+
# =============================================================================
12+
13+
""" acceptance test for resnet"""
14+
import pytest
15+
from aimet_zoo_tensorflow.mobilenetedgetpu.evaluators import mobilenet_edgetpu_quanteval
16+
17+
@pytest.mark.cuda
18+
@pytest.mark.image_classification
19+
# pylint:disable = redefined-outer-name
20+
@pytest.mark.parametrize("model_config", ["mobilenetedgetpu_w8a8"])
21+
def test_quanteval_mobilenet_edgetpu(model_config, tiny_imageNet_tfrecords):
22+
"""resnet image classification test"""
23+
24+
if tiny_imageNet_tfrecords is None:
25+
pytest.xfail(f'failed since dataset path is not set')
26+
27+
mobilenet_edgetpu_quanteval.main(
28+
[
29+
"--model-config",
30+
model_config,
31+
"--dataset-path",
32+
tiny_imageNet_tfrecords,
33+
]
34+
)

‎AcceptanceTests/tensorflow/test_mobilenet_v2_tf2_quanteval.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
@pytest.mark.cuda
1919
@pytest.mark.object_detection
2020
# pylint:disable = redefined-outer-name
21-
@pytest.mark.parametrize("model_config", ["resnet50_w8a8"])
21+
@pytest.mark.parametrize("model_config", ["mobilenetv2_w8a8"])
2222
def test_quanteval_mobilenet_v2(model_config, tiny_imageNet_root_path):
2323
"""mobilenet_v2_tf2 image classification acceptance test"""
2424

‎AcceptanceTests/tensorflow/test_ssd_mobilenetv2_quanteval.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
""" acceptance test for ssd_mobilenetv2_quanteval edgetpu"""
1414

1515
import pytest
16-
from aimet_zoo_tensorflow.ssd_mobilenet_v2.evaluators import ssd_mobilenetv2_quanteval
16+
from aimet_zoo_tensorflow.ssd_mobilenet_v2.evaluators import ssd_mobilenet_v2_quanteval
1717

1818
@pytest.mark.slow
1919
@pytest.mark.cuda
@@ -26,7 +26,7 @@ def test_quanteval_ssd_mobilenetv2(model_config, tiny_mscoco_tfrecords):
2626
if tiny_mscoco_tfrecords is None:
2727
pytest.fail(f'Dataset path is not set')
2828

29-
ssd_mobilenetv2_quanteval.main(
29+
ssd_mobilenet_v2_quanteval.main(
3030
[
3131
"--model-config",
3232
model_config,

‎AcceptanceTests/torch/CMakeLists.txt

+8-7
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,17 @@ endif (ENABLE_CUDA)
1515

1616
add_custom_target( AcceptanceTests.Torch )
1717

18-
file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/test_resnet_quanteval.py")
18+
file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/*quanteval.py")
1919
foreach(filename ${files})
20-
2120
get_filename_component( testname "${filename}" NAME_WE )
22-
message(STATUS "Testname: " testname)
23-
21+
string(SUBSTRING ${testname} 5 -1 formatted_name)
22+
message(STATUS "Now running Pytorch Test: ${formatted_name}")
23+
2424
add_custom_target(AcceptanceTests.Torch.${testname}
25-
VERBATIM COMMAND ${CMAKE_COMMAND} -E env
26-
${MZ_PYTHONPATH}
27-
${Python3_EXECUTABLE} -m pytest -s ${filename}
25+
VERBATIM COMMAND . /opt/${formatted_name}-venv/bin/activate && export HF_HOME=/opt/cache &&
26+
${CMAKE_COMMAND} -E env
27+
${MZ_PYTHONPATH}:$$PYTHONPATH
28+
python -m pytest -s ${filename}
2829
${CUDA_FLAG} --junitxml=${CMAKE_CURRENT_BINARY_DIR}/py_test_output_${testname}.xml)
2930

3031
add_dependencies( AcceptanceTests.Torch AcceptanceTests.Torch.${testname} )

‎AcceptanceTests/torch/conftest.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ def tiny_mscoco_validation_path(test_data_path):
6969
else:
7070
tiny_mscoco_validation_path = None
7171

72-
yield tiny_mscoco_validation_path.as_posix()
72+
yield tiny_mscoco_validation_path
7373

7474

7575
@pytest.fixture(scope='module')

‎AcceptanceTests/torch/pytest.ini

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,5 @@ markers =
1515
nlp: tests that belong to natual language process task
1616
object_detection: tests that belong to object detection task
1717
pose_estimation: tests that belong to pose estimation task
18-
sementic_segmentation: tests that belong to sementic segmentation task
18+
semantic_segmentation: tests that belong to sementic segmentation task
1919
super_resolution: tests that belong to super resolution task

‎AcceptanceTests/torch/staging/test_ffnet_quanteval.py

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# /usr/bin/env python3
2+
# -*- mode: python -*-
3+
# =============================================================================
4+
# @@-COPYRIGHT-START-@@
5+
#
6+
# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
7+
# Changes from QuIC are licensed under the terms and conditions at
8+
# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
9+
#
10+
# @@-COPYRIGHT-END-@@
11+
# =============================================================================
12+
""" acceptance test for ffnet semantic segmentation"""
13+
import pytest
14+
from math import isclose
15+
16+
import torch
17+
18+
from aimet_zoo_torch.ffnet.evaluators import (
19+
ffnet_quanteval,
20+
)
21+
22+
expected_results = {
23+
'segmentation_ffnet40S_dBBB_mobile': {'original_mIoU': 0.7015, 'quantized_mIoU': 0.7018},
24+
'segmentation_ffnet54S_dBBB_mobile': {'original_mIoU': 0.6957, 'quantized_mIoU': 0.7368},
25+
'segmentation_ffnet78S_BCC_mobile_pre_down': {'original_mIoU': None, 'quantized_mIoU': None},
26+
'segmentation_ffnet78S_dBBB_mobile': {'original_mIoU': 0.6904, 'quantized_mIoU': 0.6882},
27+
'segmentation_ffnet122NS_CCC_mobile_pre_down': {'original_mIoU': None, 'quantized_mIoU': None}
28+
}
29+
30+
@pytest.mark.sementic_segmentation
31+
@pytest.mark.cuda
32+
#pylint:disable = redefined-outer-name
33+
@pytest.mark.parametrize(
34+
"model_config, expected_mIoUs",[
35+
("segmentation_ffnet40S_dBBB_mobile", expected_results["segmentation_ffnet40S_dBBB_mobile"]),
36+
("segmentation_ffnet54S_dBBB_mobile", expected_results["segmentation_ffnet54S_dBBB_mobile"]),
37+
("segmentation_ffnet78S_BCC_mobile_pre_down", expected_results["segmentation_ffnet78S_BCC_mobile_pre_down"]),
38+
("segmentation_ffnet78S_dBBB_mobile", expected_results["segmentation_ffnet78S_dBBB_mobile"]),
39+
("segmentation_ffnet122NS_CCC_mobile_pre_down", expected_results["segmentation_ffnet122NS_CCC_mobile_pre_down"])
40+
]
41+
)
42+
def test_quaneval_ffnet(
43+
model_config,
44+
expected_mIoUs,
45+
tiny_cityscapes_path
46+
):
47+
"""acceptance test of hrnet for semantic segmentation"""
48+
torch.cuda.empty_cache()
49+
if tiny_cityscapes_path is None:
50+
pytest.xfail('Dataset is not set')
51+
52+
#TODO: Fix the two failing model cards
53+
if expected_mIoUs['original_mIoU'] is None:
54+
pytest.skip(f'{model_config} hasn`t passed manual testing!')
55+
56+
mIoUs = ffnet_quanteval.main(
57+
[
58+
"--model-config", model_config,
59+
"--dataset-path", tiny_cityscapes_path,
60+
"--batch-size", '2'
61+
]
62+
)
63+
64+
assert isclose(mIoUs['mIoU_orig_fp32'], expected_mIoUs['original_mIoU'], rel_tol=1e-4)
65+
assert isclose(mIoUs['mIoU_optim_int8'], expected_mIoUs['quantized_mIoU'], rel_tol=1e-4)

‎AcceptanceTests/torch/test_hrnet_image_classification_quanteval.py ‎AcceptanceTests/torch/staging/test_hrnet_image_classification_quanteval.py

File renamed without changes.

‎AcceptanceTests/torch/test_hrnet_posenet_quanteval.py ‎AcceptanceTests/torch/staging/test_hrnet_posenet_quanteval.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
@pytest.mark.pose_estimation
2020
@pytest.mark.cuda
21-
@pytest.parametrize("model_config",["hrnet_posenet_w4a8","hrnet_posenet_w8a8"])
21+
@pytest.mark.parametrize("model_config",["hrnet_posenet_w4a8","hrnet_posenet_w8a8"])
2222
def test_quaneval_hrnet_posenet(model_config, tiny_mscoco_validation_path):
2323
"""hrnet_posenet pose estimation test"""
2424
torch.cuda.empty_cache()

‎AcceptanceTests/torch/test_hrnet_sem_seg_quanteval.py ‎AcceptanceTests/torch/staging/test_hrnet_sem_seg_quanteval.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
@pytest.mark.sementic_segmentation
2424
@pytest.mark.cuda
2525
#pylint:disable = redefined-outer-name
26-
@pytest.mark.parametrize("model_config",["hrnet_sem_seg_w4a8","hrnet_sem_seg_w4a8"])
26+
@pytest.mark.parametrize("model_config",["hrnet_sem_seg_w4a8"])
2727
def test_quaneval_hrnet_sem_seg(model_config, tiny_cityscapes_path, monkeypatch):
2828
"""acceptance test of hrnet for semantic segmentation"""
2929
torch.cuda.empty_cache()

‎AcceptanceTests/torch/staging/test_inverseform_quanteval.py

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# /usr/bin/env python3
2+
# -*- mode: python -*-
3+
# =============================================================================
4+
# @@-COPYRIGHT-START-@@
5+
#
6+
# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved.
7+
# Changes from QuIC are licensed under the terms and conditions at
8+
# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf
9+
#
10+
# @@-COPYRIGHT-END-@@
11+
# =============================================================================
12+
""" acceptance test for inverseform """
13+
import pytest
14+
from math import isclose
15+
16+
from aimet_zoo_torch.inverseform.evaluators.inverseform_quanteval import main
17+
18+
expected_results = {
19+
'hrnet_16_slim_if': {'original_mIoU': 0.6883, 'quantized_mIoU': 0.6850},
20+
'ocrnet_48_if': {'original_mIoU': 0.8499, 'quantized_mIoU': 0.8465}
21+
}
22+
23+
24+
@pytest.mark.semantic_segmentation
25+
@pytest.mark.parametrize("model_config, expected_mIoUs", [('hrnet_16_slim_if', expected_results['hrnet_16_slim_if']),
26+
('ocrnet_48_if', expected_results['ocrnet_48_if'])])
27+
def test_inverseform_quanteval(
28+
model_config,
29+
expected_mIoUs,
30+
tiny_cityscapes_path
31+
):
32+
if tiny_cityscapes_path is None:
33+
pytest.xfail(f"dataset path is None!")
34+
35+
args = ['--model-config', model_config,
36+
'--dataset-path', tiny_cityscapes_path,
37+
'--batch-size', '2']
38+
mIoUs = main(args)
39+
40+
assert isclose(mIoUs['original_mIoU'], expected_mIoUs['original_mIoU'] ,rel_tol=1e-4)
41+
assert isclose(mIoUs['quantized_mIoU'], expected_mIoUs['quantized_mIoU'], rel_tol=1e-4)

‎AcceptanceTests/torch/test_minilm_quanteval.py ‎AcceptanceTests/torch/staging/test_minilm_quanteval.py

File renamed without changes.

‎AcceptanceTests/torch/test_mobilebert_quanteval.py ‎AcceptanceTests/torch/staging/test_mobilebert_quanteval.py

File renamed without changes.

0 commit comments

Comments
 (0)