diff --git a/NightlyTests/CMakeLists.txt b/AcceptanceTests/CMakeLists.txt similarity index 100% rename from NightlyTests/CMakeLists.txt rename to AcceptanceTests/CMakeLists.txt diff --git a/AcceptanceTests/tensorflow/conftest.py b/AcceptanceTests/tensorflow/conftest.py new file mode 100644 index 0000000..ba8769c --- /dev/null +++ b/AcceptanceTests/tensorflow/conftest.py @@ -0,0 +1,55 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +"""pytest configuration fixtures""" + +import pytest +import os +import warnings +from pathlib import Path + + +@pytest.fixture(scope='module') +def test_data_path(): + """ + This fixture will return the path to testing data for all models. When no + DEPENDENCY_DATA_PATH is detected, None is returned and all acceptance tests + which depend on testing data will be xfailed + """ + try: + data_path = Path(os.getenv('DEPENDENCY_DATA_PATH')) + except TypeError: + warnings.warn('In order to successfully proceed with acceptance test, please set DEPENDENCY_DATA_PATH') + data_path = None + + yield data_path + +@pytest.fixture(scope='module') +def tiny_imageNet_root_path(test_data_path): + if test_data_path is not None: + tiny_imageNet_root_path = (test_data_path / 'model_zoo_datasets/ILSVRC2012_PyTorch_reduced').as_posix() + else: + tiny_imageNet_root_path = None + + yield tiny_imageNet_root_path + + +@pytest.fixture(scope='module') +def tiny_mscoco_tfrecords(test_data_path): + if test_data_path is not None: + tiny_mscoco_tfrecords = (test_data_path / "model_zoo_datasets/tiny_mscoco_tfrecords").as_posix() + else: + tiny_mscoco_tfrecords = None + + yield tiny_mscoco_tfrecords + + diff --git a/AcceptanceTests/tensorflow/pytest.ini b/AcceptanceTests/tensorflow/pytest.ini new file mode 100644 index 0000000..ac5041c --- /dev/null +++ b/AcceptanceTests/tensorflow/pytest.ini @@ -0,0 +1,21 @@ +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +# content of pytest.ini +[pytest] +markers = + cuda: test that require CUDA to be installed + image_classification : test that belong to task of image classifcation + slow: test that runs slow + nlp: tests that belong to natual language process task + object_detection: tests that belong to object detection task + pose_estimation: tests that belong to pose estimation task + sementic_segmentation: tests that belong to sementic segmentation task + super_resolution: tests that belong to super resolution task + slow: tests that runs longer than 1 minutes per model config diff --git a/AcceptanceTests/tensorflow/test_mobiledet_edgetpu_quanteval.py b/AcceptanceTests/tensorflow/test_mobiledet_edgetpu_quanteval.py new file mode 100644 index 0000000..6ead3c8 --- /dev/null +++ b/AcceptanceTests/tensorflow/test_mobiledet_edgetpu_quanteval.py @@ -0,0 +1,41 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for mobiledet edgetpu""" + +import pytest +from aimet_zoo_tensorflow.mobiledetedgetpu.evaluators import mobiledet_edgetpu_quanteval + +@pytest.mark.slow +@pytest.mark.cuda +@pytest.mark.object_detection +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config", ["mobiledet_w8a8"]) +def test_quanteval_mobiledet_edgetpu(model_config, tiny_mscoco_tfrecords): + """mobiledet edgetpu image classification test""" + + if tiny_mscoco_tfrecords is None: + pytest.fail(f'Dataset path is not set') + + mobiledet_edgetpu_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_mscoco_tfrecords, + "--annotation-json-file", + tiny_mscoco_tfrecords+"/instances_val2017.json" + ] + ) + + + diff --git a/AcceptanceTests/tensorflow/test_mobilenet_v2_tf2_quanteval.py b/AcceptanceTests/tensorflow/test_mobilenet_v2_tf2_quanteval.py new file mode 100644 index 0000000..46a2be0 --- /dev/null +++ b/AcceptanceTests/tensorflow/test_mobilenet_v2_tf2_quanteval.py @@ -0,0 +1,38 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for mobilenet_v2_tf2_quanteval image classification""" + +import pytest +from aimet_zoo_tensorflow.mobilenet_v2_tf2.evaluators import mobilenet_v2_tf2_quanteval + +@pytest.mark.cuda +@pytest.mark.object_detection +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config", ["resnet50_w8a8"]) +def test_quanteval_mobilenet_v2(model_config, tiny_imageNet_root_path): + """mobilenet_v2_tf2 image classification acceptance test""" + + if tiny_imageNet_root_path is None: + pytest.fail(f'Dataset path is not set') + + mobilenet_v2_tf2_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_imageNet_root_path, + ] + ) + + + diff --git a/AcceptanceTests/tensorflow/test_resnet50_tf2_quanteval.py b/AcceptanceTests/tensorflow/test_resnet50_tf2_quanteval.py new file mode 100644 index 0000000..a094df7 --- /dev/null +++ b/AcceptanceTests/tensorflow/test_resnet50_tf2_quanteval.py @@ -0,0 +1,38 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for resnet50_tf2_quanteval image classification""" + +import pytest +from aimet_zoo_tensorflow.resnet50_tf2.evaluators import resnet50_tf2_quanteval + +@pytest.mark.cuda +@pytest.mark.object_detection +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config", ["resnet50_w8a8"]) +def test_quanteval_ssd_mobilenetv2(model_config, tiny_imageNet_root_path): + """resnet50_tf2 image classification acceptance test""" + + if tiny_imageNet_root_path is None: + pytest.fail(f'Dataset path is not set') + + resnet50_tf2_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_imageNet_root_path, + ] + ) + + + diff --git a/AcceptanceTests/tensorflow/test_ssd_mobilenetv2_quanteval.py b/AcceptanceTests/tensorflow/test_ssd_mobilenetv2_quanteval.py new file mode 100644 index 0000000..ffa21bc --- /dev/null +++ b/AcceptanceTests/tensorflow/test_ssd_mobilenetv2_quanteval.py @@ -0,0 +1,41 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for ssd_mobilenetv2_quanteval edgetpu""" + +import pytest +from aimet_zoo_tensorflow.ssd_mobilenet_v2.evaluators import ssd_mobilenetv2_quanteval + +@pytest.mark.slow +@pytest.mark.cuda +@pytest.mark.object_detection +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config", ["ssd_mobilenetv2_w8a8"]) +def test_quanteval_ssd_mobilenetv2(model_config, tiny_mscoco_tfrecords): + """ssd mobilenetv2 object detection acceptance test""" + + if tiny_mscoco_tfrecords is None: + pytest.fail(f'Dataset path is not set') + + ssd_mobilenetv2_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_mscoco_tfrecords, + "--annotation-json-file", + tiny_mscoco_tfrecords+"/instances_val2017.json" + ] + ) + + + diff --git a/NightlyTests/torch/CMakeLists.txt b/AcceptanceTests/torch/CMakeLists.txt similarity index 55% rename from NightlyTests/torch/CMakeLists.txt rename to AcceptanceTests/torch/CMakeLists.txt index 77ce71b..f3f9685 100644 --- a/NightlyTests/torch/CMakeLists.txt +++ b/AcceptanceTests/torch/CMakeLists.txt @@ -13,22 +13,19 @@ else (ENABLE_CUDA) set(USE_CUDA False) endif (ENABLE_CUDA) -#add_custom_target(AcceptanceTests.TorchDependencies -# COMMAND ${CMAKE_COMMAND} -E env -# "${AIMET_PYTHONPATH}" -# "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/dependencies.py ${CMAKE_CURRENT_SOURCE_DIR}/resnet18_eval_scores.csv -# ${USE_CUDA}) - add_custom_target( AcceptanceTests.Torch ) -file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/test_*.py") +file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/test_resnet_quanteval.py") foreach(filename ${files}) + get_filename_component( testname "${filename}" NAME_WE ) + message(STATUS "Testname: " testname) + add_custom_target(AcceptanceTests.Torch.${testname} VERBATIM COMMAND ${CMAKE_COMMAND} -E env - "${AIMET_PYTHONPATH}" - ${Python3_EXECUTABLE} -m pytest -s ${filename} ${CUDA_FLAG} --junitxml=${CMAKE_CURRENT_BINARY_DIR}/py_test_output_${testname}.xml) + ${MZ_PYTHONPATH} + ${Python3_EXECUTABLE} -m pytest -s ${filename} + ${CUDA_FLAG} --junitxml=${CMAKE_CURRENT_BINARY_DIR}/py_test_output_${testname}.xml) - add_dependencies( AcceptanceTests.Torch.${testname} AcceptanceTests.TorchDependencies ) add_dependencies( AcceptanceTests.Torch AcceptanceTests.Torch.${testname} ) - endforeach( filename ) + endforeach( filename ) \ No newline at end of file diff --git a/AcceptanceTests/torch/conftest.py b/AcceptanceTests/torch/conftest.py new file mode 100644 index 0000000..0518640 --- /dev/null +++ b/AcceptanceTests/torch/conftest.py @@ -0,0 +1,102 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +"""pytest configuration fixtures""" + +import pytest +import os +import warnings +from pathlib import Path + + +@pytest.fixture(scope='module') +def test_data_path(): + """ + This fixture will return the path to testing data for all models. When no + DEPENDENCY_DATA_PATH is detected, None is returned and all acceptance tests + which depend on testing data will be xfailed + """ + try: + data_path = Path(os.getenv('DEPENDENCY_DATA_PATH')) + except TypeError: + warnings.warn('In order to successfully proceed with acceptance test, please set DEPENDENCY_DATA_PATH') + data_path = None + + yield data_path + +@pytest.fixture(scope='module') +def tiny_imageNet_root_path(test_data_path): + if test_data_path is not None: + tiny_imageNet_root_path = (test_data_path / 'model_zoo_datasets/ILSVRC2012_PyTorch_reduced').as_posix() + else: + tiny_imageNet_root_path = None + + yield tiny_imageNet_root_path + + +@pytest.fixture(scope='module') +def tiny_imageNet_validation_path(test_data_path): + if test_data_path is not None: + tiny_imageNet_validation_path = (test_data_path / 'model_zoo_datasets/ILSVRC2012_PyTorch_reduced/val').as_posix() + else: + tiny_imageNet_validation_path = None + + yield tiny_imageNet_validation_path + +@pytest.fixture(scope='module') +def tiny_imageNet_train_path(test_data_path): + if test_data_path is not None: + tiny_imageNet_train_path = (test_data_path / 'model_zoo_datasets/ILSVRC2012_PyTorch_reduced/train').as_posix() + else: + tiny_imageNet_train_path = None + + yield tiny_imageNet_train_path + + +@pytest.fixture(scope='module') +def tiny_mscoco_validation_path(test_data_path): + if test_data_path is not None: + tiny_mscoco_validation_path = (test_data_path / "model_zoo_datasets/tiny_coco/val_2017").as_posix() + else: + tiny_mscoco_validation_path = None + + yield tiny_mscoco_validation_path.as_posix() + + +@pytest.fixture(scope='module') +def tiny_cityscapes_path(test_data_path): + if test_data_path is not None: + tiny_cityscapes_path = (test_data_path / "model_zoo_datasets/tiny_cityscapes").as_posix() + else: + tiny_cityscapes_path = None + + yield tiny_cityscapes_path + + +@pytest.fixture(scope='module') +def super_resolution_set5_path(test_data_path): + if test_data_path is not None: + super_resolution_set5_path = (test_data_path / "model_zoo_datasets/super_resolution_data/Set5/image_SRF_4_HR").as_posix() + else: + super_resolution_set5_path = None + + yield super_resolution_set5_path + + +@pytest.fixture(scope='module') +def PascalVOC_segmentation_test_data_path(test_data_path): + if test_data_path is not None: + pascalVOC_segmentation_path = (test_data_path / 'PascalVOCSegmentation').as_posix() + else: + pascalVOC_segmentation_path = None + + yield pascalVOC_segmentation_path diff --git a/NightlyTests/torch/hrnet.yaml b/AcceptanceTests/torch/hrnet.yaml similarity index 100% rename from NightlyTests/torch/hrnet.yaml rename to AcceptanceTests/torch/hrnet.yaml diff --git a/AcceptanceTests/torch/pytest.ini b/AcceptanceTests/torch/pytest.ini new file mode 100644 index 0000000..ba52e67 --- /dev/null +++ b/AcceptanceTests/torch/pytest.ini @@ -0,0 +1,19 @@ +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +[pytest] +markers = + cuda: test that require CUDA to be installed + image_classification : test that belong to task of image classifcation + slow: test that runs slow + nlp: tests that belong to natual language process task + object_detection: tests that belong to object detection task + pose_estimation: tests that belong to pose estimation task + sementic_segmentation: tests that belong to sementic segmentation task + super_resolution: tests that belong to super resolution task diff --git a/AcceptanceTests/torch/test_bert_quanteval.py b/AcceptanceTests/torch/test_bert_quanteval.py new file mode 100644 index 0000000..d25dfa9 --- /dev/null +++ b/AcceptanceTests/torch/test_bert_quanteval.py @@ -0,0 +1,50 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for bert NLP""" + +import pytest +import torch + +from aimet_zoo_torch.bert.evaluators import ( + bert_quanteval, +) + +@pytest.mark.nlp +@pytest.mark.cuda +#pylint:disable = redefined-outer-name +@pytest.mark.parametrize( + "model_config",[ + "bert_w8a8_cola", + "bert_w8a8_mnli", + "bert_w8a8_mrpc", + "bert_w8a8_qnli", + "bert_w8a8_qqp", + "bert_w8a8_rte", + "bert_w8a8_squad", + "bert_w8a8_sst2", + "bert_w8a8_stsb" + ] + ) +def test_quaneval_bert(model_config,monkeypatch): + """acceptance test of hrnet for semantic segmentation""" + torch.cuda.empty_cache() + # change number of evaluation samples to fewer to decrease testing time + monkeypatch.setitem(bert_quanteval.DEFAULT_CONFIG, "MAX_EVAL_SAMPLES", 2) + bert_quanteval.main( + [ + "--model_config", + model_config, + "--output_dir", + 'result' + ] + ) diff --git a/AcceptanceTests/torch/test_deeplabv3_quanteval.py b/AcceptanceTests/torch/test_deeplabv3_quanteval.py new file mode 100644 index 0000000..ad5db6b --- /dev/null +++ b/AcceptanceTests/torch/test_deeplabv3_quanteval.py @@ -0,0 +1,46 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for deeplabv3 """ + +import pytest + +from aimet_zoo_torch.deeplabv3.evaluators.deeplabv3_quanteval import main + +expected_results = { + 'dlv3_w4a8': {'original_mIoU': None, 'quantized_mIoU': None}, + 'dlv3_w8a8': {'original_mIoU': None, 'quantized_mIoU': None} +} + + +# 1. run tiny dataset through the evaluation pipeline +# 2. obtain results and compare with pre-defined numbers. If they match, we're good +# NOTE: Check data path. If None, xfail the test with messages indicating what goes wrong +# Parametrize with different model-config to make sure every config works as expected +# can set flag to enable/disable whole dataset evaluation +@pytest.mark.semantic_segmentation +@pytest.mark.parametrize("model_config, expected_results", [('dlv3_w4a8', expected_results['dlv3_w4a8']), + ('dlv3_w8a8', expected_results['dlv3_w8a8'])]) +def test_deeplabv3_quanteval( + model_config, + expected_results, + PascalVOC_segmentation_test_data_path +): + if PascalVOC_segmentation_test_data_path is None: + pytest.fail(f"dataset path is None!") + + args = ['--model-config', model_config, + '--dataset-path', PascalVOC_segmentation_test_data_path.as_posix()] + mIoUs = main(args) + + assert mIoUs['original_mIoU'] == expected_results['original_mIoU'] + assert mIoUs['quantized_mIoU'] == expected_results['quantized_mIoU'] diff --git a/AcceptanceTests/torch/test_distilbert_quanteval.py b/AcceptanceTests/torch/test_distilbert_quanteval.py new file mode 100644 index 0000000..868cc01 --- /dev/null +++ b/AcceptanceTests/torch/test_distilbert_quanteval.py @@ -0,0 +1,50 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for distilbert NLP """ + +import pytest +import torch + +from aimet_zoo_torch.distilbert.evaluators import ( + distilbert_quanteval, +) + +@pytest.mark.nlp +@pytest.mark.cuda +#pylint:disable = redefined-outer-name +@pytest.mark.parametrize( + "model_config",[ + "distilbert_w8a8_cola", + "distilbert_w8a8_mnli", + "distilbert_w8a8_mrpc", + "distilbert_w8a8_qnli", + "distilbert_w8a8_qqp", + "distilbert_w8a8_rte", + "distilbert_w8a8_squad", + "distilbert_w8a8_sst2", + "distilbert_w8a8_stsb" + ] + ) +def test_quaneval_bert(model_config,monkeypatch): + """acceptance test of distilbert for NLP""" + torch.cuda.empty_cache() + # change number of evaluation samples to fewer to decrease testing time + monkeypatch.setitem(distilbert_quanteval.DEFAULT_CONFIG, "MAX_EVAL_SAMPLES", 2) + distilbert_quanteval.main( + [ + "--model_config", + model_config, + "--output_dir", + 'result' + ] + ) diff --git a/AcceptanceTests/torch/test_efficientnetlite0_quanteval.py b/AcceptanceTests/torch/test_efficientnetlite0_quanteval.py new file mode 100644 index 0000000..1be8806 --- /dev/null +++ b/AcceptanceTests/torch/test_efficientnetlite0_quanteval.py @@ -0,0 +1,36 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for image classification""" + +import pytest +import torch +from aimet_zoo_torch.efficientnetlite0.evaluators import efficientnetlite0_quanteval + +@pytest.mark.cuda +@pytest.mark.image_classification +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config",["efficientnetlite0_w8a8"]) +def test_quanteval_efficientnetlite0_image_classification(model_config, tiny_imageNet_validation_path): + """efficientnetlite0 image classification test""" + if tiny_imageNet_validation_path is None: + pytest.fail('Dataset is not set') + torch.cuda.empty_cache() + efficientnetlite0_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_imageNet_validation_path, + ] + ) + diff --git a/AcceptanceTests/torch/test_ffnet_quanteval.py b/AcceptanceTests/torch/test_ffnet_quanteval.py new file mode 100644 index 0000000..08b33a9 --- /dev/null +++ b/AcceptanceTests/torch/test_ffnet_quanteval.py @@ -0,0 +1,46 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for ffnet semantic segmentation""" + +import pytest +import torch + +from aimet_zoo_torch.ffnet.evaluators import ( + ffnet_quanteval, +) + +@pytest.mark.sementic_segmentation +@pytest.mark.cuda +#pylint:disable = redefined-outer-name +@pytest.mark.parametrize( + "model_config",[ + "segmentation_ffnet40S_dBBB_mobile", + "segmentation_ffnet40S_dBBB_mobile", + "segmentation_ffnet78S_BCC_mobile_pre_down", + "segmentation_ffnet78S_BCC_mobile_pre_down", + "segmentation_ffnet122NS_CCC_mobile_pre_down" + ] + ) +def test_quaneval_ffnet(model_config, tiny_cityscapes_path): + """acceptance test of hrnet for semantic segmentation""" + torch.cuda.empty_cache() + if tiny_cityscapes_path is None: + pytest.fail('Dataset is not set') + ffnet_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_cityscapes_path, + ] + ) diff --git a/AcceptanceTests/torch/test_gpunet0_quanteval.py b/AcceptanceTests/torch/test_gpunet0_quanteval.py new file mode 100644 index 0000000..39f9ed3 --- /dev/null +++ b/AcceptanceTests/torch/test_gpunet0_quanteval.py @@ -0,0 +1,37 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for image classification""" + +import pytest +import torch +from aimet_zoo_torch.gpunet0.evaluator import gpunet0_quanteval + +@pytest.mark.cuda +@pytest.mark.image_classification +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config", ["gpunet0_w8a8"]) +def test_quanteval_gpunet0_image_classification(model_config, tiny_imageNet_validation_path): + """gpunet0 image classification test""" + + if tiny_imageNet_validation_path is None: + pytest.fail(f'Dataset path is not set') + + torch.cuda.empty_cache() + gpunet0_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_imageNet_validation_path, + ] + ) diff --git a/AcceptanceTests/torch/test_hrnet_image_classification_quanteval.py b/AcceptanceTests/torch/test_hrnet_image_classification_quanteval.py new file mode 100644 index 0000000..1be3b44 --- /dev/null +++ b/AcceptanceTests/torch/test_hrnet_image_classification_quanteval.py @@ -0,0 +1,36 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for image classification""" + +import pytest +import torch +from aimet_zoo_torch.hrnet_image_classification.evaluators import hrnet_image_classification_quanteval + + +@pytest.mark.image_classification +@pytest.mark.cuda +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config", ["hrnet_w32_w8a8"]) +def test_quanteval_hrnet_image_classification(model_config, tiny_imageNet_validation_path): + """hrnet image classification test""" + if tiny_imageNet_validation_path is None: + pytest.fail(f'Dataset path is not set') + torch.cuda.empty_cache() + hrnet_image_classification_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_imageNet_validation_path, + ] + ) diff --git a/NightlyTests/torch/test_pose_estimation.py b/AcceptanceTests/torch/test_hrnet_posenet_quanteval.py similarity index 68% rename from NightlyTests/torch/test_pose_estimation.py rename to AcceptanceTests/torch/test_hrnet_posenet_quanteval.py index 072a89f..d820221 100644 --- a/NightlyTests/torch/test_pose_estimation.py +++ b/AcceptanceTests/torch/test_hrnet_posenet_quanteval.py @@ -9,31 +9,25 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= -""" acceptance test for pose estimation""" + +""" acceptance test for hrnet posenet pose estimation""" + import pytest import torch from aimet_zoo_torch.hrnet_posenet.evaluators import hrnet_posenet_quanteval - -@pytest.fixture() -def model_config(): - """model config fixture""" - model_config_dict = { - "hrnet_posenet": "hrnet_posenet_w8a8", - } - return model_config_dict - - +@pytest.mark.pose_estimation @pytest.mark.cuda -def test_quaneval_hrnet_posenet(model_config, dataset_path): +@pytest.parametrize("model_config",["hrnet_posenet_w4a8","hrnet_posenet_w8a8"]) +def test_quaneval_hrnet_posenet(model_config, tiny_mscoco_validation_path): """hrnet_posenet pose estimation test""" torch.cuda.empty_cache() accuracy = hrnet_posenet_quanteval.main( [ "--model-config", - model_config["hrnet_posenet"], + model_config, "--dataset-path", - dataset_path["pose_estimation"], + tiny_mscoco_validation_path, ] ) diff --git a/NightlyTests/torch/test_semantic_segmentation.py b/AcceptanceTests/torch/test_hrnet_sem_seg_quanteval.py similarity index 56% rename from NightlyTests/torch/test_semantic_segmentation.py rename to AcceptanceTests/torch/test_hrnet_sem_seg_quanteval.py index bf34c0f..3ba7950 100644 --- a/NightlyTests/torch/test_semantic_segmentation.py +++ b/AcceptanceTests/torch/test_hrnet_sem_seg_quanteval.py @@ -9,7 +9,9 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= -""" acceptance test for semantic segmentation""" + +""" acceptance test for hrnet semantic segmentation""" + import pytest import torch @@ -17,26 +19,23 @@ hrnet_sem_seg_quanteval, ) - -@pytest.fixture() -def model_config(): - """model config fixture""" - model_config_dict = { - "hrnet_sem_seg": "hrnet_sem_seg_w8a8", - } - return model_config_dict - - +# disable this due to hrnet's hard coded image list val.lst not able to read tiny cityscapes dataset +@pytest.mark.sementic_segmentation @pytest.mark.cuda #pylint:disable = redefined-outer-name -def test_quaneval_hrnet_sem_seg(model_config, dataset_path): +@pytest.mark.parametrize("model_config",["hrnet_sem_seg_w4a8","hrnet_sem_seg_w4a8"]) +def test_quaneval_hrnet_sem_seg(model_config, tiny_cityscapes_path, monkeypatch): """acceptance test of hrnet for semantic segmentation""" torch.cuda.empty_cache() + monkeypatch.setitem(hrnet_sem_seg_quanteval.DEFAULT_CONFIG, "num_samples_cal", 2) + monkeypatch.setitem(hrnet_sem_seg_quanteval.DEFAULT_CONFIG, "num_samples_eval", 2) + if tiny_cityscapes_path is None: + pytest.fail(f'Dataset path is not set') hrnet_sem_seg_quanteval.main( [ "--model-config", - model_config["hrnet_sem_seg"], + model_config, "--dataset-path", - dataset_path["semantic_segmentation"], + tiny_cityscapes_path, ] ) diff --git a/AcceptanceTests/torch/test_minilm_quanteval.py b/AcceptanceTests/torch/test_minilm_quanteval.py new file mode 100644 index 0000000..61dad53 --- /dev/null +++ b/AcceptanceTests/torch/test_minilm_quanteval.py @@ -0,0 +1,49 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for minilm NLP """ + +import pytest +import torch + +from aimet_zoo_torch.minilm.evaluators import ( + minilm_quanteval, +) + +@pytest.mark.nlp +@pytest.mark.cuda +#pylint:disable = redefined-outer-name +@pytest.mark.parametrize( + "model_config",[ + "minilm_w8a8_cola", + "minilm_w8a8_mnli", + "minilm_w8a8_mrpc", + "minilm_w8a8_qnli", + "minilm_w8a8_qqp", + "minilm_w8a8_rte", + "minilm_w8a8_sst2", + "minilm_w8a8_stsb" + ] + ) +def test_quaneval_bert(model_config,monkeypatch): + """acceptance test of minilm for NLP""" + torch.cuda.empty_cache() + #change number of evaluation samples to fewer to decrease testing time + monkeypatch.setitem(minilm_quanteval.DEFAULT_CONFIG, "MAX_EVAL_SAMPLES", 2) + minilm_quanteval.main( + [ + "--model_config", + model_config, + "--output_dir", + 'result' + ] + ) diff --git a/AcceptanceTests/torch/test_mobilebert_quanteval.py b/AcceptanceTests/torch/test_mobilebert_quanteval.py new file mode 100644 index 0000000..fff79e9 --- /dev/null +++ b/AcceptanceTests/torch/test_mobilebert_quanteval.py @@ -0,0 +1,48 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +""" acceptance test for mobilebert NLP """ +import pytest +import torch + +from aimet_zoo_torch.mobilebert.evaluators import ( + mobilebert_quanteval, +) + +@pytest.mark.nlp +@pytest.mark.cuda +#pylint:disable = redefined-outer-name +@pytest.mark.parametrize( + "model_config",[ + "mobilebert_w8a8_cola", + "mobilebert_w8a8_mnli", + "mobilebert_w8a8_mrpc", + "mobilebert_w8a8_qnli", + "mobilebert_w8a8_qqp", + "mobilebert_w8a8_rte", + "mobilebert_w8a8_squad", + "mobilebert_w8a8_sst2", + "mobilebert_w8a8_stsb" + ] + ) +def test_quaneval_bert(model_config,monkeypatch): + """acceptance test of mobilebert for NLP""" + torch.cuda.empty_cache() + # change number of evaluation samples to fewer to decrease testing time + monkeypatch.setitem(mobilebert_quanteval.DEFAULT_CONFIG, "MAX_EVAL_SAMPLES", 2) + mobilebert_quanteval.main( + [ + "--model_config", + model_config, + "--output_dir", + 'result' + ] + ) diff --git a/AcceptanceTests/torch/test_mobilenetv2_quanteval.py b/AcceptanceTests/torch/test_mobilenetv2_quanteval.py new file mode 100644 index 0000000..613fecb --- /dev/null +++ b/AcceptanceTests/torch/test_mobilenetv2_quanteval.py @@ -0,0 +1,37 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for image classification""" + +import pytest +import torch +from aimet_zoo_torch.mobilenetv2.evaluators import mobilenetv2_quanteval + +@pytest.mark.cuda +@pytest.mark.image_classification +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config", ["mobilenetv2_w8a8"]) +def test_quanteval_mobilenetv2(model_config, tiny_imageNet_validation_path): + """mobilenetv2 image classification test""" + + if tiny_imageNet_validation_path is None: + pytest.fail(f'Dataset path is not set') + + torch.cuda.empty_cache() + mobilenetv2_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_imageNet_validation_path, + ] + ) diff --git a/AcceptanceTests/torch/test_mobilevit_quanteval.py b/AcceptanceTests/torch/test_mobilevit_quanteval.py new file mode 100644 index 0000000..5c59fd8 --- /dev/null +++ b/AcceptanceTests/torch/test_mobilevit_quanteval.py @@ -0,0 +1,38 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for mobilevit image classification""" + +import pytest +import torch +from aimet_zoo_torch.mobilevit.evaluators import mobilevit_quanteval + +@pytest.mark.image_classification +@pytest.mark.cuda +@pytest.mark.parametrize("model_config", ["mobilevit_w8a8"]) +# pylint:disable = redefined-outer-name +def test_quanteval_mobilevit_image_classification(model_config, tiny_imageNet_validation_path, tiny_imageNet_train_path): + """mobilevit image classification test""" + if tiny_imageNet_validation_path is None: + pytest.fail(f'Dataset path is not set') + + torch.cuda.empty_cache() + mobilevit_quanteval.main( + [ + "--model_config", + model_config, + "--train_dir", + tiny_imageNet_train_path, + "--validation_dir", + tiny_imageNet_validation_path + ] + ) diff --git a/AcceptanceTests/torch/test_quicksrnet_quanteval.py b/AcceptanceTests/torch/test_quicksrnet_quanteval.py new file mode 100644 index 0000000..f06f094 --- /dev/null +++ b/AcceptanceTests/torch/test_quicksrnet_quanteval.py @@ -0,0 +1,52 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for quicksrnet""" + +import pytest +import torch +from aimet_zoo_torch.quicksrnet.evaluators import quicksrnet_quanteval + + +@pytest.mark.cuda +@pytest.mark.object_detection +@pytest.mark.parametrize( + "model_config",[ + "quicksrnet_small_1.5x_w8a8", + "quicksrnet_small_2x_w8a8", + "quicksrnet_small_3x_w8a8", + "quicksrnet_small_4x_w8a8", + "quicksrnet_medium_1.5x_w8a8", + "quicksrnet_medium_2x_w8a8", + "quicksrnet_medium_3x_w8a8", + "quicksrnet_medium_4x_w8a8", + "quicksrnet_large_1.5x_w8a8", + "quicksrnet_large_2x_w8a8", + "quicksrnet_large_3x_w8a8", + "quicksrnet_large_4x_w8a8", + "quicksrnet_large_4x_w4a8" + ] + ) +# pylint:disable = redefined-outer-name +def test_quaneval_quicksrnet(model_config, super_resolution_set5_path): + """quicksrnet super resolution acceptance test""" + if super_resolution_set5_path is None: + pytest.fail(f'Dataset path is not set') + torch.cuda.empty_cache() + quicksrnet_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + super_resolution_set5_path, + ] + ) diff --git a/NightlyTests/torch/test_super_resolution.py b/AcceptanceTests/torch/test_regnet_quanteval.py similarity index 56% rename from NightlyTests/torch/test_super_resolution.py rename to AcceptanceTests/torch/test_regnet_quanteval.py index e01ae93..3379efb 100644 --- a/NightlyTests/torch/test_super_resolution.py +++ b/AcceptanceTests/torch/test_regnet_quanteval.py @@ -9,31 +9,29 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= -""" acceptance test for super resolution""" -import pytest -import torch -from aimet_zoo_torch.quicksrnet.evaluators import quicksrnet_quanteval +""" acceptance test for regnet""" -@pytest.fixture() -def model_config(): - """model config fixture""" - model_config_dict = { - "quicksrnet": "quicksrnet_small_1.5x_w8a8", - } - return model_config_dict - +import pytest +import torch +from aimet_zoo_torch.regnet.evaluator import regnet_quanteval @pytest.mark.cuda +@pytest.mark.image_classification # pylint:disable = redefined-outer-name -def test_quaneval_quicksrnet(model_config, dataset_path): - """quicksrnet super resolution acceptance test""" +@pytest.mark.parametrize("model_config", ["regnet_x_3_2gf_w8a8"]) +def test_quanteval_regnet(model_config, tiny_imageNet_validation_path): + """regnet image classification test""" + if tiny_imageNet_validation_path is None: + pytest.fail(f'Dataset path is not set') + torch.cuda.empty_cache() - quicksrnet_quanteval.main( + regnet_quanteval.main( [ "--model-config", - model_config["quicksrnet"], + model_config, "--dataset-path", - dataset_path["super_resolution"], + tiny_imageNet_validation_path, ] ) + diff --git a/NightlyTests/torch/test_image_classification.py b/AcceptanceTests/torch/test_resnet_quanteval.py similarity index 63% rename from NightlyTests/torch/test_image_classification.py rename to AcceptanceTests/torch/test_resnet_quanteval.py index 4a52f78..e1bb34d 100644 --- a/NightlyTests/torch/test_image_classification.py +++ b/AcceptanceTests/torch/test_resnet_quanteval.py @@ -9,31 +9,27 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= -""" acceptance test for image classification""" +""" acceptance test for resnet""" import pytest import torch from aimet_zoo_torch.resnet.evaluator import resnet_quanteval - -@pytest.fixture() -def model_config(): - """model config fixture""" - model_config_dict = { - "resnet18": "resnet18_w8a8", - } - return model_config_dict - - @pytest.mark.cuda +@pytest.mark.image_classification # pylint:disable = redefined-outer-name -def test_quanteval_resnet18(model_config, dataset_path): - """resnet18 image classification test""" +@pytest.mark.parametrize("model_config", ["resnet18_w8a8","resnet50_w8a8","resnet101_w8a8"]) +def test_quanteval_resnet(model_config, tiny_imageNet_validation_path): + """resnet image classification test""" + + if tiny_imageNet_validation_path is None: + pytest.fail(f'Dataset path is not set') + torch.cuda.empty_cache() resnet_quanteval.main( [ "--model-config", - model_config["resnet18"], + model_config, "--dataset-path", - dataset_path["image_classification"], + tiny_imageNet_validation_path, ] ) diff --git a/AcceptanceTests/torch/test_resnext_quanteval.py b/AcceptanceTests/torch/test_resnext_quanteval.py new file mode 100644 index 0000000..ac9ba52 --- /dev/null +++ b/AcceptanceTests/torch/test_resnext_quanteval.py @@ -0,0 +1,40 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for resnext image classification""" + +import pytest +import torch +from aimet_zoo_torch.resnext.evaluator import resnext_quanteval + + +@pytest.mark.image_classification +@pytest.mark.cuda +@pytest.mark.parametrize("model_config",["resnext101_w8a8"]) +# pylint:disable = redefined-outer-name +def test_quanteval_resnext(model_config, tiny_imageNet_validation_path): + """resnext image classification test""" + if tiny_imageNet_validation_path is None: + pytest.fail('Dataset is not set') + + torch.cuda.empty_cache() + resnext_quanteval.main( + [ + "--model-config", + model_config["resnext"], + "--dataset-path", + tiny_imageNet_validation_path, + ] + ) + + + diff --git a/AcceptanceTests/torch/test_roberta_quanteval.py b/AcceptanceTests/torch/test_roberta_quanteval.py new file mode 100644 index 0000000..a1829e0 --- /dev/null +++ b/AcceptanceTests/torch/test_roberta_quanteval.py @@ -0,0 +1,49 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for roberta NLP """ + +import pytest +import torch + +from aimet_zoo_torch.roberta.evaluators import ( + roberta_quanteval, +) + +@pytest.mark.nlp +@pytest.mark.cuda +#pylint:disable = redefined-outer-name +@pytest.mark.parametrize( + "model_config",[ + "roberta_w8a8_cola", + "roberta_w8a8_mnli", + "roberta_w8a8_mrpc", + "roberta_w8a8_qnli", + "roberta_w8a8_qqp", + "roberta_w8a8_rte", + "roberta_w8a8_sst2", + "roberta_w8a8_stsb" + ] + ) +def test_quaneval_bert(model_config,monkeypatch): + """acceptance test of roberta for NLP""" + torch.cuda.empty_cache() + # change number of evaluation samples to fewer to decrease testing time + monkeypatch.setitem(roberta_quanteval.DEFAULT_CONFIG, "MAX_EVAL_SAMPLES", 2) + roberta_quanteval.main( + [ + "--model_config", + model_config, + "--output_dir", + 'result' + ] + ) diff --git a/AcceptanceTests/torch/test_ssd_mobilenetv2_quanteval.py b/AcceptanceTests/torch/test_ssd_mobilenetv2_quanteval.py new file mode 100644 index 0000000..334903a --- /dev/null +++ b/AcceptanceTests/torch/test_ssd_mobilenetv2_quanteval.py @@ -0,0 +1,37 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for ssd_mobilenetv2 object detection""" + +import pytest +import torch +from aimet_zoo_torch.ssd_mobilenetv2.evaluators import ssd_mobilenetv2_quanteval + +@pytest.mark.object_detection +@pytest.mark.cuda +@pytest.mark.parametrize("model_config",["ssd_mobilenetv2_w8a8"]) +def test_quaneval_ssd_mobilenetv2(model_config, PascalVOC_segmentation_test_data_path, monkeypatch): + monkeypatch.setitem(ssd_mobilenetv2_quanteval.DEFAULT_CONFIG, "num_samples_cal", 1) + monkeypatch.setitem(ssd_mobilenetv2_quanteval.DEFAULT_CONFIG, "num_samples_eval", 1) + torch.cuda.empty_cache() + if PascalVOC_segmentation_test_data_path is None: + pytest.fail('Dataset not set') + ssd_mobilenetv2_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + PascalVOC_segmentation_test_data_path, + ] + ) + + diff --git a/AcceptanceTests/torch/test_ssd_res50_quanteval.py b/AcceptanceTests/torch/test_ssd_res50_quanteval.py new file mode 100644 index 0000000..e259fa1 --- /dev/null +++ b/AcceptanceTests/torch/test_ssd_res50_quanteval.py @@ -0,0 +1,36 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for ssd_res50 object detection""" + +import pytest +import torch +from aimet_zoo_torch.ssd_res50.evaluators import ssd_res50_quanteval + +#some issues with the code , failed as SIT test results +@pytest.mark.cuda +@pytest.mark.object_detection +@pytest.mark.parametrize("model_config",["ssd_res50_w8a8"]) +def test_quaneval_ssd_res50(model_config, tiny_mscoco_validation_path): + torch.cuda.empty_cache() + if tiny_mscoco_validation_path is None: + pytest.fail('Dataset not set') + ssd_res50_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_mscoco_validation_path, + ] + ) + + diff --git a/AcceptanceTests/torch/test_uniformer_classification_quanteval.py b/AcceptanceTests/torch/test_uniformer_classification_quanteval.py new file mode 100644 index 0000000..0c4c051 --- /dev/null +++ b/AcceptanceTests/torch/test_uniformer_classification_quanteval.py @@ -0,0 +1,37 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for uniformer image classification""" + +import pytest +import torch +from aimet_zoo_torch.uniformer_classification.evaluators import uniformer_classification_quanteval + +@pytest.mark.cuda +@pytest.mark.image_classification +# pylint:disable = redefined-outer-name +@pytest.mark.parametrize("model_config", ["uniformer_classification_w8a8"]) +def test_quanteval_resnet(model_config, tiny_imageNet_root_path): + """resnet image classification test""" + + if tiny_imageNet_root_path is None: + pytest.fail(f'dataset path is not set') + + torch.cuda.empty_cache() + uniformer_classification_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_imageNet_root_path, + ] + ) diff --git a/AcceptanceTests/torch/test_vit_quanteval.py b/AcceptanceTests/torch/test_vit_quanteval.py new file mode 100644 index 0000000..64051ed --- /dev/null +++ b/AcceptanceTests/torch/test_vit_quanteval.py @@ -0,0 +1,37 @@ +# /usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" acceptance test for vit image classification""" + +import pytest +import torch +from aimet_zoo_torch.vit.evaluators import vit_quanteval + +@pytest.mark.image_classification +@pytest.mark.cuda +@pytest.mark.parametrize("model_config", ["vit_w8a8"]) +# pylint:disable = redefined-outer-name +def test_quanteval_vit_image_classification(model_config, tiny_imageNet_validation_path, tiny_imageNet_train_path): + """vit image classification test""" + torch.cuda.empty_cache() + if tiny_imageNet_validation_path is None: + pytest.fail('Dataset not set') + vit_quanteval.main( + [ + "--model_config", + model_config, + "--train_dir", + tiny_imageNet_train_path, + "--validation_dir", + tiny_imageNet_validation_path + ] + ) diff --git a/NightlyTests/torch/test_object_detection.py b/AcceptanceTests/torch/test_yolox_quanteval.py similarity index 53% rename from NightlyTests/torch/test_object_detection.py rename to AcceptanceTests/torch/test_yolox_quanteval.py index 6ca1e58..4e216ed 100644 --- a/NightlyTests/torch/test_object_detection.py +++ b/AcceptanceTests/torch/test_yolox_quanteval.py @@ -9,30 +9,26 @@ # # @@-COPYRIGHT-END-@@ # ============================================================================= -""" acceptance test for object detection""" + +""" acceptance test for yolox object detection""" + import pytest import torch from aimet_zoo_torch.yolox.evaluators import yolox_quanteval - -@pytest.fixture() -def model_config(): - model_config_dict = { - "yolox": "yolox_s", - } - return model_config_dict - - +@pytest.mark.object_detection @pytest.mark.cuda -def test_quaneval_yolox(model_config, dataset_path): - torch.cuda.empty_cache() - - yolox_quanteval.main( - [ - "--model-config", - model_config["yolox"], - "--dataset-path", - dataset_path["object_detection"], - ] - ) +@pytest.mark.parametrize("model_config",["yolox_s","yolox_l"]) +def test_quaneval_yolox(model_config, tiny_mscoco_validation_path): + torch.cuda.empty_cache() + if tiny_mscoco_validation_path is None: + pytest.fail('Dataset path is not set') + yolox_quanteval.main( + [ + "--model-config", + model_config, + "--dataset-path", + tiny_mscoco_validation_path, + ] + ) diff --git a/AcceptanceTests/torch/voc-model-labels.txt b/AcceptanceTests/torch/voc-model-labels.txt new file mode 100644 index 0000000..5e16a90 --- /dev/null +++ b/AcceptanceTests/torch/voc-model-labels.txt @@ -0,0 +1,21 @@ +BACKGROUND +aeroplane +bicycle +bird +boat +bottle +bus +car +cat +chair +cow +diningtable +dog +horse +motorbike +person +pottedplant +sheep +sofa +train +tvmonitor \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 9dcfcb5..0b472b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,30 @@ cmake_minimum_required(VERSION 3.17) project(aimet-model-zoo) +# ------------------------------- +# Conditional build for CUDA +# ------------------------------- +if (NOT (DEFINED ENABLE_CUDA)) + message("Compiling with CUDA not explicitly disabled. Enabling implicitly") + set(ENABLE_CUDA ON CACHE BOOL "") + +endif(NOT (DEFINED ENABLE_CUDA)) + +message("Cuda: " ${ENABLE_CUDA}) + +find_package(Python3 COMPONENTS Interpreter Development) +message("Found python: ${Python3_FOUND}, at ${Python3_LIBRARIES}") + +find_package(PkgConfig) +pkg_search_module(LAPACKE REQUIRED lapacke) + +if(NOT DEFINED MZ_PYTHONPATH) + set(MZ_PYTHONPATH "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}:${CMAKE_BINARY_DIR}/artifacts" CACHE STRING "python path") +endif() +set(MZ_PYTHONPATH "${MZ_PYTHONPATH}:${CMAKE_CURRENT_SOURCE_DIR}/TrainingExtensions/common/src/python") + +set(ENV{PYTHONPATH} "${CMAKE_CURRENT_SOURCE_DIR}") + # Set the software version from version.txt file (if not already set) if(NOT DEFINED SW_VERSION) file(STRINGS "packaging/version.txt" SW_VERSION) @@ -90,6 +114,13 @@ add_custom_target(pylintmodelzoo COMMAND ${CMAKE_COMMAND} -DAIMET_PACKAGE_PATH=${AIMET_PACKAGE_PATH} -DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} -DENABLE_TENSORFLOW=${ENABLE_TENSORFLOW} -DENABLE_TORCH=${ENABLE_TORCH} -DSW_VERSION=${SW_VERSION} -DPROJECT_NAME=${CMAKE_PROJECT_NAME} -P ${CMAKE_CURRENT_SOURCE_DIR}/packaging/pylint_model_zoo.cmake ) +# ------------------------------- +# Acceptance tests +# ------------------------------- + +add_subdirectory(AcceptanceTests) + + # ------------------------------- # Deployment # ------------------------------- diff --git a/NOTICE.txt b/NOTICE.txt index 33dd9b3..58508d8 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -588,3 +588,35 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. # ------------------------------------------------------------------------------ + +========================================================================== +Copyright 2022 SenseTime X-Lab. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +========================================================================== + +# ------------------------------------------------------------------------------ +Copyright 2018-2023 OpenMMLab. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +# ------------------------------------------------------------------------------ diff --git a/NightlyTests/torch/conftest.py b/NightlyTests/torch/conftest.py deleted file mode 100644 index 2bd96ef..0000000 --- a/NightlyTests/torch/conftest.py +++ /dev/null @@ -1,34 +0,0 @@ -import pytest -from pathlib import Path - - -@pytest.fixture(scope='module') -def data_folder(tmp_path_factory): - """ - This fixture will return a shortcut path for saved data. When there's no - such shortcut path, a tmp path will be generated. Use this with discretion - because anything stored in a tmp path will be gone. Set DEPENDENCY_DATA_PATH - only when you have permanent files stored for testing - """ - if is_cache_env_set(): - dependency_path = Path(os.getenv('DEPENDENCY_DATA_PATH')) - else: - dependency_path = None - - data_path = dependency_path if (dependency_path and dependency_path.exists()) else tmp_path_factory.mktemp('data') - - yield data_path - - -@pytest.fixture(autouse=True) -def dataset_path(data_path): - """this fixture return the dataset paths for acceptance tests""" - - dataset_path = { - "image_classification":str(data_path)+"ILSVRC2012_PyTorch_reduced/val", - "object_detection":str(data_path)+"tiny_coco/val_2017", - "pose_estimation":str(data_path)+"tiny_coco/val_2017", - "semantic_segmentation":str(data_path)+"cityscapes", - "super_reslution":str(data_path)+"/super_resolution_data/Set5/image_SRF_4_HR" - } - return dataset_path diff --git a/NightlyTests/torch/pytest.ini b/NightlyTests/torch/pytest.ini deleted file mode 100644 index 360046b..0000000 --- a/NightlyTests/torch/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -# content of pytest.ini -[pytest] -markers = - cuda: test that require CUDA to be installed \ No newline at end of file diff --git a/README.md b/README.md index 29d5920..d777424 100755 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ An original FP32 source model is quantized either using post-training quantizati W4A8[7] - Image Classification + Image Classification MobileNetV2 GitHub Repo Pretrained Model @@ -147,6 +147,16 @@ An original FP32 source model is quantized either using post-training quantizati 78.86 78.42 TBD + + + Uniformer + Repo + Prepared Models + Quantized Models + (ImageNet dataset) Accuracy + 82.9 + 81.9 + TBD Object Detection @@ -332,6 +342,17 @@ An original FP32 source model is quantized either using post-training quantizati 50.59% 50.58% + + Video Understanding + mmaction2 BMN + GitHub Repo + Pretrained Model + Quantized Model + (ActivityNet) auc + 67.25 + 67.05 + TBD + Speech Recognition DeepSpeech2 diff --git a/aimet_zoo_tensorflow/mobiledetedgetpu/evaluators/mobiledet_edgetpu_quanteval.py b/aimet_zoo_tensorflow/mobiledetedgetpu/evaluators/mobiledet_edgetpu_quanteval.py index c82ca80..9b2b67e 100644 --- a/aimet_zoo_tensorflow/mobiledetedgetpu/evaluators/mobiledet_edgetpu_quanteval.py +++ b/aimet_zoo_tensorflow/mobiledetedgetpu/evaluators/mobiledet_edgetpu_quanteval.py @@ -32,7 +32,7 @@ session = InteractiveSession(config=config) -def arguments(): +def arguments(raw_args): """argument parser""" parser = argparse.ArgumentParser( description="Evaluation script for TensorFlow MobileDet-EdgeTPU." @@ -57,7 +57,7 @@ def arguments(): parser.add_argument( "--use-cuda", help="Run evaluation on GPU", type=bool, default=True ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args @@ -71,9 +71,9 @@ def __init__(self, args): setattr(self, arg, getattr(args, arg)) -def main(): +def main(raw_args=None): """Evaluation main function""" - args = arguments() + args = arguments(raw_args) config = ModelConfig(args) dataloader = get_dataloader( diff --git a/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/mobilenet_v2_tf2_quanteval.py b/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/mobilenet_v2_tf2_quanteval.py index 8a4efe4..9ddae03 100644 --- a/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/mobilenet_v2_tf2_quanteval.py +++ b/aimet_zoo_tensorflow/mobilenet_v2_tf2/evaluators/mobilenet_v2_tf2_quanteval.py @@ -9,11 +9,11 @@ # ============================================================================= ''' do TF2 mobilenetv2 quantization and evaluation''' import argparse -import tensorflow as tf +import tensorflow as tf from aimet_zoo_tensorflow.mobilenet_v2_tf2.model.model_definition import MobilenetV2 from aimet_zoo_tensorflow.mobilenet_v2_tf2.evaluators.eval_func import get_eval_func -def arguments(): +def arguments(raw_args): ''' parses command line arguments ''' @@ -24,16 +24,16 @@ def arguments(): parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) parser.add_argument('--batch-size', help='batch_size for loading data', type=int, default=16) parser.add_argument('--use-cuda', help='Run evaluation on GPU', type=bool, default=True) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args -def main(): +def main(raw_args=None): """ Run evaluation """ gpu_devices = tf.config.experimental.list_physical_devices("GPU") for device in gpu_devices: tf.config.experimental.set_memory_growth(device, True) - args = arguments() + args = arguments(raw_args) # Evaluation function eval_func = get_eval_func(dataset_dir=args.dataset_path, diff --git a/aimet_zoo_tensorflow/resnet50_tf2/ResNet50_TF2.md b/aimet_zoo_tensorflow/resnet50_tf2/ResNet50_TF2.md index 70a1baf..22fecf2 100755 --- a/aimet_zoo_tensorflow/resnet50_tf2/ResNet50_TF2.md +++ b/aimet_zoo_tensorflow/resnet50_tf2/ResNet50_TF2.md @@ -1,7 +1,7 @@ # Tensorflow ResNet50 for TensorFlow 2.4 ## Setup AI Model Efficiency Toolkit (AIMET) -Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.25/packaging/install.md) before proceeding further. This evaluation was run using [AIMET 1.25 for TensorFlow 2.4](https://github.com/quic/aimet/releases/tag/1.25) i.e. please set `release_tag="1.25"` and `AIMET_VARIANT="tf_gpu"` in the above instructions. +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.26/packaging/install.md) before proceeding further. This evaluation was run using [AIMET 1.26 for TensorFlow 2.4](https://github.com/quic/aimet/releases/tag/1.26) i.e. please set `release_tag="1.26"` and `AIMET_VARIANT="tf_gpu"` in the above instructions. ## Additional Dependencies pip install numpy==1.19.5 diff --git a/aimet_zoo_tensorflow/resnet50_tf2/evaluators/resnet50_tf2_quanteval.py b/aimet_zoo_tensorflow/resnet50_tf2/evaluators/resnet50_tf2_quanteval.py index edd2d1e..6641c28 100755 --- a/aimet_zoo_tensorflow/resnet50_tf2/evaluators/resnet50_tf2_quanteval.py +++ b/aimet_zoo_tensorflow/resnet50_tf2/evaluators/resnet50_tf2_quanteval.py @@ -13,7 +13,7 @@ from aimet_zoo_tensorflow.resnet50_tf2.model.model_definition import Resnet50 from aimet_zoo_tensorflow.resnet50_tf2.evaluators.eval_func import get_eval_func -def arguments(): +def arguments(raw_args): ''' parses command line arguments ''' @@ -24,12 +24,12 @@ def arguments(): parser.add_argument('--default-param-bw', help='Default parameter bitwidth for quantization.', type=int, default=8) parser.add_argument('--batch-size', help='batch_size for loading data', type=int, default=16) parser.add_argument('--use-cuda', help='Run evaluation on GPU', type=bool, default=True) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args -def main(): +def main(raw_args=None): """ Run evaluation """ - args = arguments() + args = arguments(raw_args) gpu_devices = tf.config.experimental.list_physical_devices("GPU") for device in gpu_devices: diff --git a/aimet_zoo_tensorflow/resnet50_tf2/model/model_cards/resnet50_w8a8.json b/aimet_zoo_tensorflow/resnet50_tf2/model/model_cards/resnet50_w8a8.json index 5afdd3f..19f8752 100755 --- a/aimet_zoo_tensorflow/resnet50_tf2/model/model_cards/resnet50_w8a8.json +++ b/aimet_zoo_tensorflow/resnet50_tf2/model/model_cards/resnet50_w8a8.json @@ -18,7 +18,7 @@ "url_pre_opt_weights": null, "url_post_opt_weights": null, "url_adaround_encodings": null, - "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/tensorflow2-resnet50/resnet50_w8a8.encodings", + "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/tensorflow2_resnet50/resnet50_w8a8.encodings", "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.25/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" } } diff --git a/aimet_zoo_tensorflow/ssd_mobilenet_v2/evaluators/ssd_mobilenet_v2_quanteval.py b/aimet_zoo_tensorflow/ssd_mobilenet_v2/evaluators/ssd_mobilenet_v2_quanteval.py index b94706d..10e0137 100644 --- a/aimet_zoo_tensorflow/ssd_mobilenet_v2/evaluators/ssd_mobilenet_v2_quanteval.py +++ b/aimet_zoo_tensorflow/ssd_mobilenet_v2/evaluators/ssd_mobilenet_v2_quanteval.py @@ -32,7 +32,7 @@ session = InteractiveSession(config=config) -def arguments(): +def arguments(raw_args): """argument parser""" parser = argparse.ArgumentParser( description="Evaluation script for TensorFlow SSD-MobileNetV2." @@ -57,7 +57,7 @@ def arguments(): parser.add_argument( "--use-cuda", help="Run evaluation on GPU", type=bool, default=True ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args @@ -71,9 +71,9 @@ def __init__(self, args): setattr(self, arg, getattr(args, arg)) -def main(): +def main(raw_args=None): """Evaluation main function""" - args = arguments() + args = arguments(raw_args) config = ModelConfig(args) dataloader = get_dataloader( diff --git a/aimet_zoo_torch/bert/dataloader/dataloaders.py b/aimet_zoo_torch/bert/dataloader/dataloaders.py index 6c3c247..035cc65 100644 --- a/aimet_zoo_torch/bert/dataloader/dataloaders.py +++ b/aimet_zoo_torch/bert/dataloader/dataloaders.py @@ -303,7 +303,9 @@ def preprocess_function(examples): return datasets -def eval_function(model,tokenizer,datasets,data_args,training_args): +def eval_function(model,tokenizer,datasets,data_args,training_args,max_eval_samples=None): + if max_eval_samples is not None: + data_args.max_eval_samples = max(len(datasets), max_eval_samples) ## case 1. when dataset is glue if hasattr(data_args,'task_name'): train_dataset = datasets["train"] @@ -366,10 +368,19 @@ def compute_metrics(p: EvalPrediction): # Loop to handle MNLI double evaluation (matched, mis-matched) tasks = [data_args.task_name] - eval_datasets = [eval_dataset] + + if data_args.max_eval_samples is not None: + # We will select subset of samples (number of samples = max_eval_samples) from the whole dataset + eval_dataset = eval_dataset.select( + range(data_args.max_eval_samples)) + eval_datasets = [eval_dataset] if data_args.task_name == "mnli": tasks.append("mnli-mm") - eval_datasets.append(datasets["validation_mismatched"]) + mnli_mm_eval_dataset = datasets["validation_mismatched"] + if data_args.max_eval_samples is not None: + mnli_mm_eval_dataset = mnli_mm_eval_dataset.select( + range(data_args.max_eval_samples)) + eval_datasets.append(mnli_mm_eval_dataset) for eval_dataset, _ in zip(eval_datasets, tasks): eval_result = trainer.evaluate(eval_dataset=eval_dataset) eval_results.update(eval_result) diff --git a/aimet_zoo_torch/bert/evaluators/bert_quanteval.py b/aimet_zoo_torch/bert/evaluators/bert_quanteval.py index f7bc340..f4a5819 100644 --- a/aimet_zoo_torch/bert/evaluators/bert_quanteval.py +++ b/aimet_zoo_torch/bert/evaluators/bert_quanteval.py @@ -33,7 +33,7 @@ from aimet_zoo_torch.bert.dataloader import get_datasets, eval_function -def parse_args(): +def parse_args(raw_args): """argument parser""" parser = argparse.ArgumentParser( description="Evaluating Bert model on GLUE datasets" @@ -53,24 +53,26 @@ def parse_args(): "--output_dir", type=str, default=None, + required=True, help="Output directory", - ) - args = parser.parse_args() + ) + args = parser.parse_args(raw_args) for arg in vars(args): print("{:30s} : {}".format(arg, getattr(args, arg))) return args +DEFAULT_CONFIG = {"MAX_EVAL_SAMPLES": None} -def main(): +def main(raw_args=None): """main function for quantization evaluation""" - args = parse_args() + args = parse_args(raw_args) logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO, ) - model = Bert(model_config=args.model_config) + model = Bert(model_config=args.model_config,args=raw_args) # get original model and tokenizer model_orig, tokenizer = model.from_pretrained() @@ -84,17 +86,19 @@ def main(): tokenizer=tokenizer, ) + logging.info("*** original model evaluation ***") + # evaluation of original model original_eval_results = eval_function( - model_orig, tokenizer, datasets, model.data_args, model.training_args + model_orig, tokenizer, datasets, model.data_args, model.training_args, max_eval_samples= DEFAULT_CONFIG['MAX_EVAL_SAMPLES'] ) - + logging.info("*** getting quantsim ***") # get quantsim object quantsim_model = model.get_quantsim() - + logging.info("*** evaluation quantsim model ***") # evaluation of quantsim model optimized_eval_results = eval_function( - quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args + quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args, max_eval_samples= DEFAULT_CONFIG['MAX_EVAL_SAMPLES'] ) logging.info(f"***** Original Eval results *****") diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_cola.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_cola.json index ca091dd..dc8fd6d 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_cola.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_cola.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mnli.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mnli.json index 8e88bae..8c974ac 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mnli.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mnli.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mrpc.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mrpc.json index 587ae70..25a8b3b 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mrpc.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_mrpc.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qnli.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qnli.json index ea85270..3a41588 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qnli.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qnli.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qqp.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qqp.json index b3313ef..5577136 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qqp.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_qqp.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_rte.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_rte.json index 671a6a2..5dc736b 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_rte.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_rte.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_squad.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_squad.json index 5abdb66..aceb54b 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_squad.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_squad.json @@ -30,8 +30,8 @@ "use_auth_token":false }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_sst2.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_sst2.json index af8e539..c87a671 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_sst2.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_sst2.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_stsb.json b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_stsb.json index c15cb66..982fc5f 100644 --- a/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_stsb.json +++ b/aimet_zoo_torch/bert/model/model_cards/bert_w8a8_stsb.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/bert/model/model_definition.py b/aimet_zoo_torch/bert/model/model_definition.py index 776bf19..88d0010 100644 --- a/aimet_zoo_torch/bert/model/model_definition.py +++ b/aimet_zoo_torch/bert/model/model_definition.py @@ -33,7 +33,7 @@ class Bert(Downloader): """model Bert configuration class""" #pylint:disable = import-outside-toplevel - def __init__(self, model_config=None): + def __init__(self, model_config=None, args=None): if model_config == "bert_w8a8_squad": from aimet_zoo_torch.bert.model.utils.utils_qa_dataclass import ( ModelArguments, @@ -46,10 +46,12 @@ def __init__(self, model_config=None): DataTrainingArguments, AuxArguments, ) - parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) + self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = defaultdict(lambda: None) if model_config: - config_filepath = parent_dir + "/model_cards/" + model_config + ".json" + config_filepath = os.path.join( + self.parent_dir, "model_cards", model_config + ".json" + ) with open(config_filepath) as f_in: self.cfg = json.load(f_in) Downloader.__init__( @@ -57,7 +59,7 @@ def __init__(self, model_config=None): url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], - model_dir=parent_dir, + model_dir=self.parent_dir, ) # Parse arguments parser = HfArgumentParser( @@ -68,13 +70,14 @@ def __init__(self, model_config=None): data_args, training_args, aux_args, - ) = parser.parse_args_into_dataclasses() - + ) = parser.parse_args_into_dataclasses(args) self.model = None self.model_args = model_args self.data_args = data_args self.training_args = training_args self.aux_args = aux_args + self.aux_args.fmodel_path = os.path.join(self.parent_dir, self.aux_args.fmodel_path) + self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) # additional setup of the argsumetns from model_config if model_config == "bert_w8a8_squad": self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] @@ -85,8 +88,10 @@ def __init__(self, model_config=None): self.training_args.do_eval = True # setup the download path from arguments self.path_pre_opt_weights = self.aux_args.fmodel_path + self.path_post_opt_weights = self.aux_args.qmodel_path - + + def from_pretrained(self): """get original or optimized model Parameters: diff --git a/aimet_zoo_torch/bert/model/utils/utils_nlclassifier_dataclass.py b/aimet_zoo_torch/bert/model/utils/utils_nlclassifier_dataclass.py index 6170df4..8770bf1 100644 --- a/aimet_zoo_torch/bert/model/utils/utils_nlclassifier_dataclass.py +++ b/aimet_zoo_torch/bert/model/utils/utils_nlclassifier_dataclass.py @@ -126,11 +126,11 @@ class AuxArguments: Auxiliary arguments pertaining to training. """ fmodel_path: str = field( - default="../model/weights/fp.pth", + default="weights/fp.pth", metadata={"help": "Path to the full-precision model"} ) qmodel_path: str = field( - default="../model/weights/qat.ckpt", + default="weights/qat.ckpt", metadata={"help": "Path to the quantized model"} ) model_config: str = field( diff --git a/aimet_zoo_torch/bert/model/utils/utils_qa_dataclass.py b/aimet_zoo_torch/bert/model/utils/utils_qa_dataclass.py index 2d089c1..bf7a72b 100644 --- a/aimet_zoo_torch/bert/model/utils/utils_qa_dataclass.py +++ b/aimet_zoo_torch/bert/model/utils/utils_qa_dataclass.py @@ -66,11 +66,11 @@ class AuxArguments: Auxiliary arguments pertaining to training. """ fmodel_path: str = field( - default="../model/weights/fp.pth", + default="weights/fp.pth", metadata={"help": "Path to the full-precision model"} ) qmodel_path: str = field( - default="../model/weights/qat.ckpt", + default="weights/qat.ckpt", metadata={"help": "Path to the quantized model"} ) model_config: str = field( diff --git a/aimet_zoo_torch/common/downloader.py b/aimet_zoo_torch/common/downloader.py index 1a31a6d..8a6a70f 100644 --- a/aimet_zoo_torch/common/downloader.py +++ b/aimet_zoo_torch/common/downloader.py @@ -17,6 +17,7 @@ from urllib.request import urlretrieve import urllib.request import progressbar +import requests import gdown# pylint: disable=import-error @@ -98,9 +99,14 @@ def __init__( if self.path_zipped_checkpoint else None ) + # GITHUB TOKEN for internal use cases + self.GITHUB_TOKEN = None + self.INTERNAL_REPO_URL = None def _download_from_url(self, src: str, dst: str, show_progress=False): """Receives a source URL or path and a storage destination path, evaluates the source, fetches the file, and stores at the destination""" + # import pdb + # pdb.set_trace() if not os.path.exists(self._download_storage_path): os.makedirs(self._download_storage_path) if src is None: @@ -108,10 +114,13 @@ def _download_from_url(self, src: str, dst: str, show_progress=False): if src.startswith("https://drive.google.com"): gdown.download(url=src, output=dst, quiet=True, verify=False) elif src.startswith("http"): - if show_progress: - urlretrieve(src, dst, DownloadProgressBar()) + if 'qualcomm' in src: + self._download_from_internal(src,dst) else: - urlretrieve(src, dst) + if show_progress: + urlretrieve(src, dst, DownloadProgressBar()) + else: + urlretrieve(src, dst) else: assert os.path.exists( src @@ -119,6 +128,61 @@ def _download_from_url(self, src: str, dst: str, show_progress=False): copy2(src, dst) return None + def _convert_src_to_asset_url(self, src: str): + """convert src url to asset url + """ + # 0. get release_tag and file_name from url + release_tag, file_name = self._find_tag(src) + # 1. read all release in to all_releases + headers = { + 'Authorization': 'token ' + self.GITHUB_TOKEN , + 'Accept': 'application/json', + } + + resp = requests.get(self.INTERNAL_REPO_URL,headers = headers,timeout=(4, 30)) + + all_releases = resp.json() + # 2. check if release_tag in all_releases else report artifacts not uploade + content_with_tag_name = [s for s in all_releases if s['tag_name']== release_tag ] + if content_with_tag_name is None: + raise NameError('this release tag is not uploaded, check if release tag or if this release is uploaded yet') + # 3. check if file_name in all_releases['release_tag'], else report file not uploaded or file name is wrong + assets_with_tag_name = content_with_tag_name[0]['assets'] + asset_with_file_name = [s for s in assets_with_tag_name if s['name']== file_name ] + if asset_with_file_name is None: + raise NameError('this artifact is not uploaded or naming has mismatch with release') + # 4. return asset_url + return asset_with_file_name[0]['url'] + + def _find_tag(self, src: str): + """find out release tag and file name + /download/tensorflow2_resnet50/resnet50_w8a8.encodings + return should be + tensorflow2_resnet50, resnet50_w8a8.encodings + """ + url_breakdown = src.split('/') + return url_breakdown[-2], url_breakdown[-1] + + def _download_from_internal(self, src: str, dst: str): + """Use GITHUB_TOKEN evironment variable to download from internal github repo link + + """ + self.GITHUB_TOKEN= os.getenv("GITHUB_TOKEN") + self.INTERNAL_REPO_URL= os.getenv("INTERNAL_REPO_URL") + if self.GITHUB_TOKEN is None: + raise NameError("GITHUB_TOKEN not setup, not able to download from internal github url, exit program!") + if self.INTERNAL_REPO_URL is None: + raise NameError("variable INTERNAL_REPO_URL not setup, use export INTERNAL_REPO_URL= to setup before continuing") + asset_url = self._convert_src_to_asset_url(src) + headers = { + 'Authorization': 'token ' + self.GITHUB_TOKEN , + 'Accept': 'application/octet-stream', + } + resp = requests.get(asset_url,headers = headers, timeout=(4, 30) ) + with open(dst, 'wb') as file: + file.write(resp.content) + + def _download_pre_opt_weights(self, show_progress=False): """downloads pre optimization weights""" self._download_from_url( diff --git a/aimet_zoo_torch/distilbert/dataloader/dataloaders.py b/aimet_zoo_torch/distilbert/dataloader/dataloaders.py index 1658158..8274732 100644 --- a/aimet_zoo_torch/distilbert/dataloader/dataloaders.py +++ b/aimet_zoo_torch/distilbert/dataloader/dataloaders.py @@ -303,7 +303,9 @@ def preprocess_function(examples): return datasets -def eval_function(model,tokenizer,datasets,data_args,training_args): +def eval_function(model,tokenizer,datasets,data_args,training_args,max_eval_samples=None): + if max_eval_samples is not None: + data_args.max_eval_samples = max_eval_samples ## case 1. when dataset is glue if hasattr(data_args,'task_name'): train_dataset = datasets["train"] @@ -366,10 +368,18 @@ def compute_metrics(p: EvalPrediction): # Loop to handle MNLI double evaluation (matched, mis-matched) tasks = [data_args.task_name] - eval_datasets = [eval_dataset] + if data_args.max_eval_samples is not None: + # We will select sample from whole data + eval_dataset = eval_dataset.select( + range(data_args.max_eval_samples)) + eval_datasets = [eval_dataset] if data_args.task_name == "mnli": tasks.append("mnli-mm") - eval_datasets.append(datasets["validation_mismatched"]) + mnli_mm_eval_dataset = datasets["validation_mismatched"] + if data_args.max_eval_samples is not None: + mnli_mm_eval_dataset = mnli_mm_eval_dataset.select( + range(data_args.max_eval_samples)) + eval_datasets.append(mnli_mm_eval_dataset) for eval_dataset, _ in zip(eval_datasets, tasks): eval_result = trainer.evaluate(eval_dataset=eval_dataset) eval_results.update(eval_result) diff --git a/aimet_zoo_torch/distilbert/evaluators/distilbert_quanteval.py b/aimet_zoo_torch/distilbert/evaluators/distilbert_quanteval.py index a84343c..a165c7a 100644 --- a/aimet_zoo_torch/distilbert/evaluators/distilbert_quanteval.py +++ b/aimet_zoo_torch/distilbert/evaluators/distilbert_quanteval.py @@ -32,7 +32,7 @@ from aimet_zoo_torch.distilbert.dataloader import get_datasets, eval_function -def parse_args(): +def parse_args(raw_args): """argument parser""" parser = argparse.ArgumentParser(description="Evaluating DistilBert model ") parser.add_argument( @@ -56,22 +56,23 @@ def parse_args(): default=None, help="Output directory", ) - args = parser.parse_args() + args = parser.parse_args(raw_args) for arg in vars(args): print("{:30s} : {}".format(arg, getattr(args, arg))) return args +DEFAULT_CONFIG = {"MAX_EVAL_SAMPLES": None} -def main(): +def main(raw_args=None): """main function for quantization evaluation""" - args = parse_args() + args = parse_args(raw_args) logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO, ) - model = DistilBert(model_config=args.model_config) + model = DistilBert(model_config=args.model_config,args=raw_args) # get original model and tokenizer model_orig, tokenizer = model.get_model_from_pretrained() @@ -87,7 +88,7 @@ def main(): # evaluation of original model original_eval_results = eval_function( - model_orig, tokenizer, datasets, model.data_args, model.training_args + model_orig, tokenizer, datasets, model.data_args, model.training_args, max_eval_samples=DEFAULT_CONFIG['MAX_EVAL_SAMPLES'] ) # get quantsim object @@ -95,7 +96,7 @@ def main(): # evaluation of quantsim model optimized_eval_results = eval_function( - quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args + quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args, max_eval_samples=DEFAULT_CONFIG['MAX_EVAL_SAMPLES'] ) logging.info(f"***** Original Eval results *****") diff --git a/aimet_zoo_torch/distilbert/model/baseline_models/distilbert/modeling_distilbert.py b/aimet_zoo_torch/distilbert/model/baseline_models/distilbert/modeling_distilbert.py index 94681c1..756e270 100644 --- a/aimet_zoo_torch/distilbert/model/baseline_models/distilbert/modeling_distilbert.py +++ b/aimet_zoo_torch/distilbert/model/baseline_models/distilbert/modeling_distilbert.py @@ -41,7 +41,7 @@ from torch.nn import CrossEntropyLoss from aimet_torch import elementwise_ops -from aimet_zoo_torch.distilbert.utils.activations import get_activation +from aimet_zoo_torch.distilbert.model.utils.activations import get_activation from transformers.file_utils import ( add_code_sample_docstrings, diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_cola.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_cola.json index 6cc9b73..8fb10d0 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_cola.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_cola.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mnli.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mnli.json index 361ec31..8ed2c86 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mnli.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mnli.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mrpc.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mrpc.json index f84fd96..d1b3294 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mrpc.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_mrpc.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qnli.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qnli.json index 5225dcf..47c6346 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qnli.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qnli.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qqp.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qqp.json index d4c8d24..ac88591 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qqp.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_qqp.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_rte.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_rte.json index 5e3c98d..6e9635c 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_rte.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_rte.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_squad.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_squad.json index e9cee9e..f7b52d9 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_squad.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_squad.json @@ -30,8 +30,8 @@ "use_auth_token":false }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_sst2.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_sst2.json index 340a6fb..dd8d6a8 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_sst2.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_sst2.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_stsb.json b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_stsb.json index f96e3a6..6d2dbf7 100644 --- a/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_stsb.json +++ b/aimet_zoo_torch/distilbert/model/model_cards/distilbert_w8a8_stsb.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/distilbert/model/model_definition.py b/aimet_zoo_torch/distilbert/model/model_definition.py index 59d55aa..f25ee73 100644 --- a/aimet_zoo_torch/distilbert/model/model_definition.py +++ b/aimet_zoo_torch/distilbert/model/model_definition.py @@ -25,15 +25,15 @@ from transformers import AutoConfig, AutoTokenizer, TrainingArguments # transformers import -from aimet_zoo_torch.distilbert.model.baseline_models import distilbert +from aimet_zoo_torch.distilbert.model import baseline_models from aimet_zoo_torch.common.downloader import Downloader -sys.modules["baseline_models.distilbert"] = distilbert +sys.modules["baseline_models"] = baseline_models class DistilBert(Downloader): """model distilbert configuration class""" #pylint:disable = import-outside-toplevel - def __init__(self, model_config=None): + def __init__(self, model_config=None, args=None): if model_config == "distilbert_w8a8_squad": from aimet_zoo_torch.distilbert.model.utils.utils_qa_dataclass import ( ModelArguments, @@ -46,10 +46,12 @@ def __init__(self, model_config=None): DataTrainingArguments, AuxArguments, ) - parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) + self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = defaultdict(lambda: None) if model_config: - config_filepath = parent_dir + "/model_cards/" + model_config + ".json" + config_filepath = os.path.join( + self.parent_dir, "model_cards", model_config + ".json" + ) with open(config_filepath) as f_in: self.cfg = json.load(f_in) Downloader.__init__( @@ -57,7 +59,7 @@ def __init__(self, model_config=None): url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], - model_dir=parent_dir, + model_dir=self.parent_dir, ) # Parse arguments parser = HfArgumentParser( @@ -68,13 +70,15 @@ def __init__(self, model_config=None): data_args, training_args, aux_args, - ) = parser.parse_args_into_dataclasses() + ) = parser.parse_args_into_dataclasses(args) self.model = None self.model_args = model_args self.data_args = data_args self.training_args = training_args self.aux_args = aux_args + self.aux_args.fmodel_path = os.path.join(self.parent_dir, self.aux_args.fmodel_path) + self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) # additional setup of the argsumetns from model_config if model_config == "distilbert_w8a8_squad": self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] diff --git a/aimet_zoo_torch/distilbert/model/utils/utils_nlclassifier_dataclass.py b/aimet_zoo_torch/distilbert/model/utils/utils_nlclassifier_dataclass.py index efe3ed8..51f1548 100644 --- a/aimet_zoo_torch/distilbert/model/utils/utils_nlclassifier_dataclass.py +++ b/aimet_zoo_torch/distilbert/model/utils/utils_nlclassifier_dataclass.py @@ -125,11 +125,11 @@ class AuxArguments: Auxiliary arguments pertaining to training. """ fmodel_path: str = field( - default="../model/weights/fp.pth", + default="weights/fp.pth", metadata={"help": "Path to the full-precision model"} ) qmodel_path: str = field( - default="../model/weights/qat.ckpt", + default="weights/qat.ckpt", metadata={"help": "Path to the quantized model"} ) model_config: str = field( diff --git a/aimet_zoo_torch/distilbert/model/utils/utils_qa_dataclass.py b/aimet_zoo_torch/distilbert/model/utils/utils_qa_dataclass.py index 56a0a80..ea24f1a 100644 --- a/aimet_zoo_torch/distilbert/model/utils/utils_qa_dataclass.py +++ b/aimet_zoo_torch/distilbert/model/utils/utils_qa_dataclass.py @@ -66,11 +66,11 @@ class AuxArguments: Auxiliary arguments pertaining to training. """ fmodel_path: str = field( - default="../model/weights/fp.pth", + default="weights/fp.pth", metadata={"help": "Path to the full-precision model"} ) qmodel_path: str = field( - default="../model/weights/qat.ckpt", + default="weights/qat.ckpt", metadata={"help": "Path to the quantized model"} ) model_config: str = field( diff --git a/aimet_zoo_torch/efficientnetlite0/evaluators/efficientnetlite0_quanteval.py b/aimet_zoo_torch/efficientnetlite0/evaluators/efficientnetlite0_quanteval.py index aff0d45..dfa6988 100755 --- a/aimet_zoo_torch/efficientnetlite0/evaluators/efficientnetlite0_quanteval.py +++ b/aimet_zoo_torch/efficientnetlite0/evaluators/efficientnetlite0_quanteval.py @@ -20,7 +20,7 @@ from aimet_zoo_torch.efficientnetlite0 import EfficientNetLite0 -def arguments(): +def arguments(raw_args): """add arguments""" parser = argparse.ArgumentParser( description="0725 changed script for efficientnet_lite0 quantization" @@ -49,7 +49,7 @@ def arguments(): parser.add_argument( "--use-cuda", help="Run evaluation on GPU", type=bool, default=True ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args @@ -88,10 +88,10 @@ def __init__(self, args): setattr(self, arg, getattr(args, arg)) -def main(): +def main(raw_args=None): """ main evaluation function""" #pylint:disable = no-member - args = arguments() + args = arguments(raw_args) config = ModelConfig(args) seed(seednum=23, use_cuda=args.use_cuda) diff --git a/aimet_zoo_torch/gpunet0/evaluator/gpunet0_quanteval.py b/aimet_zoo_torch/gpunet0/evaluator/gpunet0_quanteval.py index 6c2b5c9..2cc8164 100755 --- a/aimet_zoo_torch/gpunet0/evaluator/gpunet0_quanteval.py +++ b/aimet_zoo_torch/gpunet0/evaluator/gpunet0_quanteval.py @@ -30,7 +30,7 @@ def seed(seed_number): torch.cuda.manual_seed_all(seed_number) -def arguments(): +def arguments(raw_args): """argument parser""" # pylint: disable = redefined-outer-name parser = argparse.ArgumentParser( @@ -55,13 +55,14 @@ def arguments(): parser.add_argument( "--use-cuda", help="Run evaluation on GPU.", type=bool, default=True ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args -def main(args): +def main(raw_args=None): """ " main function for quantization evaluation""" # pylint: disable = redefined-outer-name + args = arguments(raw_args) seed(1234) evaluator = evaluate( testBatch=args.batch_size, @@ -111,5 +112,4 @@ def main(args): if __name__ == "__main__": - args = arguments() - main(args) + main() diff --git a/aimet_zoo_torch/hrnet_image_classification/evaluators/hrnet_image_classification_quanteval.py b/aimet_zoo_torch/hrnet_image_classification/evaluators/hrnet_image_classification_quanteval.py index bd808f0..14c90e3 100644 --- a/aimet_zoo_torch/hrnet_image_classification/evaluators/hrnet_image_classification_quanteval.py +++ b/aimet_zoo_torch/hrnet_image_classification/evaluators/hrnet_image_classification_quanteval.py @@ -21,7 +21,7 @@ from aimet_zoo_torch.common.utils.image_net_data_loader import ImageNetDataLoader -def arguments(): +def arguments(raw_args): """parse command line arguments""" parser = argparse.ArgumentParser(description="Evaluation script for HRNet") parser.add_argument( @@ -39,16 +39,16 @@ def arguments(): parser.add_argument( "--use-cuda", help="Use GPU for evaluation", default=True, type=bool ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args -def main(): +def main(raw_args=None): """run evaluator""" #pylint:disable = undefined-variable, logging-fstring-interpolation - args = arguments() + args = arguments(raw_args) logging.basicConfig(level=logging.DEBUG) - logger = logging.getLogger(_name_) + logger = logging.getLogger(__name__) device = get_device(args) # get imagenet validation dataloader eval_dataloader = ImageNetDataLoader( diff --git a/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/hrnet_sem_seg_quanteval.py b/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/hrnet_sem_seg_quanteval.py index 71c3669..dbc03ad 100644 --- a/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/hrnet_sem_seg_quanteval.py +++ b/aimet_zoo_torch/hrnet_semantic_segmentation/evaluators/hrnet_sem_seg_quanteval.py @@ -66,6 +66,8 @@ def __init__(self, args): for arg in vars(args): setattr(self, arg, getattr(args, arg)) +DEFAULT_CONFIG = {"num_samples_cal": 2000, "num_samples_eval": None} + def main(raw_args=None): """run evaluator""" @@ -76,8 +78,8 @@ def main(raw_args=None): # Load quantized model, compute encodings and evaluate model = HRNetSemSeg(model_config=args.model_config) sim = model.get_quantsim(quantized=True) - eval_func_calibration = model_eval(config, num_samples=2000) - eval_func = model_eval(config) + eval_func_calibration = model_eval(config, num_samples=DEFAULT_CONFIG['num_samples_cal']) + eval_func = model_eval(config,num_samples=DEFAULT_CONFIG['num_samples_eval']) sim.compute_encodings( forward_pass_callback=eval_func_calibration, forward_pass_callback_args=config ) diff --git a/aimet_zoo_torch/minilm/dataloader/dataloaders.py b/aimet_zoo_torch/minilm/dataloader/dataloaders.py index a8b6e80..6e9e9a7 100644 --- a/aimet_zoo_torch/minilm/dataloader/dataloaders.py +++ b/aimet_zoo_torch/minilm/dataloader/dataloaders.py @@ -303,7 +303,9 @@ def preprocess_function(examples): return datasets -def eval_function(model,tokenizer,datasets,data_args,training_args): +def eval_function(model,tokenizer,datasets,data_args,training_args, max_eval_samples=None): + if max_eval_samples is not None: + data_args.max_eval_samples = max_eval_samples ## case 1. when dataset is glue if hasattr(data_args,'task_name'): train_dataset = datasets["train"] @@ -366,10 +368,18 @@ def compute_metrics(p: EvalPrediction): # Loop to handle MNLI double evaluation (matched, mis-matched) tasks = [data_args.task_name] - eval_datasets = [eval_dataset] + if data_args.max_eval_samples is not None: + # We will select sample from whole data + eval_dataset = eval_dataset.select( + range(data_args.max_eval_samples)) + eval_datasets = [eval_dataset] if data_args.task_name == "mnli": tasks.append("mnli-mm") - eval_datasets.append(datasets["validation_mismatched"]) + mnli_mm_eval_dataset = datasets["validation_mismatched"] + if data_args.max_eval_samples is not None: + mnli_mm_eval_dataset = mnli_mm_eval_dataset.select( + range(data_args.max_eval_samples)) + eval_datasets.append(mnli_mm_eval_dataset) for eval_dataset, _ in zip(eval_datasets, tasks): eval_result = trainer.evaluate(eval_dataset=eval_dataset) eval_results.update(eval_result) diff --git a/aimet_zoo_torch/minilm/evaluators/minilm_quanteval.py b/aimet_zoo_torch/minilm/evaluators/minilm_quanteval.py index 0374ff6..10f2900 100644 --- a/aimet_zoo_torch/minilm/evaluators/minilm_quanteval.py +++ b/aimet_zoo_torch/minilm/evaluators/minilm_quanteval.py @@ -32,7 +32,7 @@ from aimet_zoo_torch.minilm.dataloader import get_datasets, eval_function -def parse_args(): +def parse_args(raw_args): """argument parser""" parser = argparse.ArgumentParser( description="Evaluating MiniLM model on GLUE datasets" @@ -54,22 +54,23 @@ def parse_args(): default=None, help="Output directory", ) - args = parser.parse_args() + args = parser.parse_args(raw_args) for arg in vars(args): print("{:30s} : {}".format(arg, getattr(args, arg))) return args +DEFAULT_CONFIG = {"MAX_EVAL_SAMPLES": None} -def main(): +def main(raw_args=None): """main function for quantization evaluation""" - args = parse_args() + args = parse_args(raw_args) logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO, ) - model = Minilm(model_config=args.model_config) + model = Minilm(model_config=args.model_config,args=raw_args) # get original model and tokenizer model_orig, tokenizer = model.get_model_from_pretrained() @@ -85,7 +86,7 @@ def main(): # evaluation of original model original_eval_results = eval_function( - model_orig, tokenizer, datasets, model.data_args, model.training_args + model_orig, tokenizer, datasets, model.data_args, model.training_args, max_eval_samples=DEFAULT_CONFIG['MAX_EVAL_SAMPLES'] ) # get quantsim object @@ -93,7 +94,7 @@ def main(): # evaluation of quantsim model optimized_eval_results = eval_function( - quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args + quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args, max_eval_samples=DEFAULT_CONFIG['MAX_EVAL_SAMPLES'] ) logging.info(f"***** Original Eval results *****") diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_cola.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_cola.json index 8a3ed5a..5ccb42d 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_cola.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_cola.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mnli.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mnli.json index 8d04fe4..d3ff32a 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mnli.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mnli.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mrpc.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mrpc.json index 699fd5f..b32b513 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mrpc.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_mrpc.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qnli.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qnli.json index 7451b01..fbc9ddf 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qnli.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qnli.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qqp.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qqp.json index 6c26996..5f1cd64 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qqp.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_qqp.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_rte.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_rte.json index 1e58e59..619ffee 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_rte.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_rte.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_squad.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_squad.json index 6a70ee6..ea21abd 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_squad.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_squad.json @@ -30,8 +30,8 @@ "use_auth_token":false }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_sst2.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_sst2.json index 650786f..dbb88f8 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_sst2.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_sst2.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_stsb.json b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_stsb.json index 6ef4354..e38eea9 100644 --- a/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_stsb.json +++ b/aimet_zoo_torch/minilm/model/model_cards/minilm_w8a8_stsb.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/minilm/model/model_definition.py b/aimet_zoo_torch/minilm/model/model_definition.py index cb881de..e30b2a1 100644 --- a/aimet_zoo_torch/minilm/model/model_definition.py +++ b/aimet_zoo_torch/minilm/model/model_definition.py @@ -34,7 +34,7 @@ class Minilm(Downloader): """model minilm configuration class""" #pylint:disable = import-outside-toplevel - def __init__(self, model_config=None): + def __init__(self, model_config=None,args=None): if model_config == "minilm_w8a8_squad": from aimet_zoo_torch.minilm.model.utils.utils_qa_dataclass import ( ModelArguments, @@ -47,10 +47,12 @@ def __init__(self, model_config=None): DataTrainingArguments, AuxArguments, ) - parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) + self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = defaultdict(lambda: None) if model_config: - config_filepath = parent_dir + "/model_cards/" + model_config + ".json" + config_filepath = os.path.join( + self.parent_dir, "model_cards", model_config + ".json" + ) with open(config_filepath) as f_in: self.cfg = json.load(f_in) Downloader.__init__( @@ -58,7 +60,7 @@ def __init__(self, model_config=None): url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], - model_dir=parent_dir, + model_dir=self.parent_dir, ) # Parse arguments parser = HfArgumentParser( @@ -69,13 +71,15 @@ def __init__(self, model_config=None): data_args, training_args, aux_args, - ) = parser.parse_args_into_dataclasses() + ) = parser.parse_args_into_dataclasses(args) self.model = None self.model_args = model_args self.data_args = data_args self.training_args = training_args self.aux_args = aux_args + self.aux_args.fmodel_path = os.path.join(self.parent_dir, self.aux_args.fmodel_path) + self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) # additional setup of the argsumetns from model_config if model_config == "minilm_w8a8_squad": self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] diff --git a/aimet_zoo_torch/minilm/model/utils/utils_nlclassifier_dataclass.py b/aimet_zoo_torch/minilm/model/utils/utils_nlclassifier_dataclass.py index 9b75cc5..7b207ee 100644 --- a/aimet_zoo_torch/minilm/model/utils/utils_nlclassifier_dataclass.py +++ b/aimet_zoo_torch/minilm/model/utils/utils_nlclassifier_dataclass.py @@ -124,11 +124,11 @@ class AuxArguments: Auxiliary arguments pertaining to training. """ fmodel_path: str = field( - default="../model/weights/fp.pth", + default="weights/fp.pth", metadata={"help": "Path to the full-precision model"} ) qmodel_path: str = field( - default="../model/weights/qat.ckpt", + default="weights/qat.ckpt", metadata={"help": "Path to the quantized model"} ) model_config: str = field( diff --git a/aimet_zoo_torch/minilm/model/utils/utils_qa_dataclass.py b/aimet_zoo_torch/minilm/model/utils/utils_qa_dataclass.py index 9a447c9..aee4279 100644 --- a/aimet_zoo_torch/minilm/model/utils/utils_qa_dataclass.py +++ b/aimet_zoo_torch/minilm/model/utils/utils_qa_dataclass.py @@ -67,11 +67,11 @@ class AuxArguments: Auxiliary arguments pertaining to training. """ fmodel_path: str = field( - default="../model/weights/fp.pth", + default="weights/fp.pth", metadata={"help": "Path to the full-precision model"} ) qmodel_path: str = field( - default="../model/weights/qat.ckpt", + default="weights/qat.ckpt", metadata={"help": "Path to the quantized model"} ) model_config: str = field( diff --git a/aimet_zoo_torch/mmaction2/__init__.py b/aimet_zoo_torch/mmaction2/__init__.py new file mode 100644 index 0000000..46b27eb --- /dev/null +++ b/aimet_zoo_torch/mmaction2/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +""" package for getting mmaction2 original model and quantized model""" +from .runner.loops import AIMETTestLoop +from .evaluators.metrics import AIMETANetMetric diff --git a/aimet_zoo_torch/mmaction2/evaluators/__init__.py b/aimet_zoo_torch/mmaction2/evaluators/__init__.py new file mode 100644 index 0000000..01720c8 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/evaluators/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= diff --git a/aimet_zoo_torch/mmaction2/evaluators/metrics/__init__.py b/aimet_zoo_torch/mmaction2/evaluators/metrics/__init__.py new file mode 100644 index 0000000..62542f8 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/evaluators/metrics/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +from .anet_metric import AIMETANetMetric diff --git a/aimet_zoo_torch/mmaction2/evaluators/metrics/anet_metric.py b/aimet_zoo_torch/mmaction2/evaluators/metrics/anet_metric.py new file mode 100644 index 0000000..ae07c94 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/evaluators/metrics/anet_metric.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# pylint: skip-file +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +# Copyright (c) OpenMMLab. All rights reserved. +import os +import os.path as osp +from collections import OrderedDict +from typing import Any, Optional, Sequence, Tuple + +import mmcv +import mmengine +import numpy as np +from mmengine.evaluator import BaseMetric + +from mmaction.evaluation import average_recall_at_avg_proposals +from mmaction.registry import METRICS +from mmaction.utils import ConfigType + + +@METRICS.register_module() +class AIMETANetMetric(BaseMetric): + """ActivityNet dataset evaluation metric.""" + + def __init__(self, + metric_type: str = 'TEM', + collect_device: str = 'cpu', + prefix: Optional[str] = None, + metric_options: dict = {}, + dump_config: ConfigType = dict(out='')): + super().__init__(collect_device=collect_device, prefix=prefix) + self.metric_type = metric_type + + assert 'out' in dump_config + self.output_format = dump_config.pop('output_format', 'csv') + self.out = dump_config['out'] + + self.metric_options = metric_options + if self.metric_type == 'AR@AN': + self.ground_truth = {} + + def process(self, *args, **kwargs): + pass + + def process_original(self, data_batch: Sequence[Tuple[Any, dict]], + predictions: Sequence[dict]) -> None: + """Process one batch of data samples and predictions. The processed + results should be stored in ``self.results``, which will be used to + compute the metrics when all batches have been processed. + + Args: + data_batch (Sequence[Tuple[Any, dict]]): A batch of data + from the dataloader. + predictions (Sequence[dict]): A batch of outputs from + the model. + """ + for pred in predictions: + self.results.append(pred) + + if self.metric_type == 'AR@AN': + data_batch = data_batch[1] + for data_sample in data_batch: + video_info = data_sample.metainfo + video_id = video_info['video_name'][2:] + this_video_gt = [] + for ann in video_info['annotations']: + t_start, t_end = ann['segment'] + label = ann['label'] + this_video_gt.append([t_start, t_end, label]) + self.ground_truth[video_id] = np.array(this_video_gt) + + def process_dummy(self, data_batch: Sequence[Tuple[Any, dict]], + predictions: Sequence[dict]) -> None: + """Process one batch of data samples and predictions. The processed + results should be stored in ``self.results``, which will be used to + compute the metrics when all batches have been processed. + + Args: + data_batch (Sequence[Tuple[Any, dict]]): A batch of data + from the dataloader. + predictions (Sequence[dict]): A batch of outputs from + the model. + """ + return + + def compute_metrics(self, results: list) -> dict: + """Compute the metrics from processed results. + + If `metric_type` is 'TEM', only dump middle results and do not compute + any metrics. + Args: + results (list): The processed results of each batch. + Returns: + dict: The computed metrics. The keys are the names of the metrics, + and the values are corresponding results. + """ + self.dump_results(results) + if self.metric_type == 'AR@AN': + return self.compute_ARAN(results) + return OrderedDict() + + def compute_ARAN(self, *args, **kwargs): + pass + + def compute_ARAN_original(self, results: list) -> dict: + """AR@AN evaluation metric.""" + temporal_iou_thresholds = self.metric_options.setdefault( + 'AR@AN', {}).setdefault('temporal_iou_thresholds', + np.linspace(0.5, 0.95, 10)) + max_avg_proposals = self.metric_options.setdefault( + 'AR@AN', {}).setdefault('max_avg_proposals', 100) + if isinstance(temporal_iou_thresholds, list): + temporal_iou_thresholds = np.array(temporal_iou_thresholds) + + eval_results = OrderedDict() + proposal, num_proposals = self._import_proposals(results) + + recall, _, _, auc = average_recall_at_avg_proposals( + self.ground_truth, + proposal, + num_proposals, + max_avg_proposals=max_avg_proposals, + temporal_iou_thresholds=temporal_iou_thresholds) + eval_results['auc'] = auc + eval_results['AR@1'] = np.mean(recall[:, 0]) + eval_results['AR@5'] = np.mean(recall[:, 4]) + eval_results['AR@10'] = np.mean(recall[:, 9]) + eval_results['AR@100'] = np.mean(recall[:, 99]) + + return eval_results + + def compute_ARAN_dummy(self, results: list) -> dict: + """AR@AN evaluation metric.""" + eval_results = OrderedDict() + eval_results['auc'] = 0 + eval_results['AR@1'] = 0 + eval_results['AR@5'] = 0 + eval_results['AR@10'] = 0 + eval_results['AR@100'] = 0 + + return eval_results + + def dump_results(self, results, version='VERSION 1.3'): + """Save middle or final results to disk.""" + if self.output_format == 'json': + result_dict = self.proposals2json(results) + output_dict = { + 'version': version, + 'results': result_dict, + 'external_data': {} + } + mmengine.dump(output_dict, self.out) + elif self.output_format == 'csv': + os.makedirs(self.out, exist_ok=True) + header = 'action,start,end,tmin,tmax' + for result in results: + video_name, outputs = result + output_path = osp.join(self.out, video_name + '.csv') + np.savetxt( + output_path, + outputs, + header=header, + delimiter=',', + comments='') + else: + raise ValueError( + f'The output format {self.output_format} is not supported.') + + @staticmethod + def proposals2json(results, show_progress=False): + """Convert all proposals to a final dict(json) format. + Args: + results (list[dict]): All proposals. + show_progress (bool): Whether to show the progress bar. + Defaults: False. + Returns: + dict: The final result dict. E.g. + .. code-block:: Python + dict(video-1=[dict(segment=[1.1,2.0]. score=0.9), + dict(segment=[50.1, 129.3], score=0.6)]) + """ + result_dict = {} + print('Convert proposals to json format') + if show_progress: + prog_bar = mmcv.ProgressBar(len(results)) + for result in results: + video_name = result['video_name'] + result_dict[video_name[2:]] = result['proposal_list'] + if show_progress: + prog_bar.update() + return result_dict + + @staticmethod + def _import_proposals(results): + """Read predictions from results.""" + proposals = {} + num_proposals = 0 + for result in results: + video_id = result['video_name'][2:] + this_video_proposals = [] + for proposal in result['proposal_list']: + t_start, t_end = proposal['segment'] + score = proposal['score'] + this_video_proposals.append([t_start, t_end, score]) + num_proposals += 1 + proposals[video_id] = np.array(this_video_proposals) + return proposals, num_proposals + +@METRICS.register_module() +class ANetMetric_dummy(BaseMetric): + """ActivityNet dataset evaluation metric.""" + + def __init__(self, + metric_type: str = 'TEM', + collect_device: str = 'cpu', + prefix: Optional[str] = None, + metric_options: dict = {}, + dump_config: ConfigType = dict(out='')): + super().__init__(collect_device=collect_device, prefix=prefix) + self.metric_type = metric_type + + assert 'out' in dump_config + self.output_format = dump_config.pop('output_format', 'csv') + self.out = dump_config['out'] + + self.metric_options = metric_options + if self.metric_type == 'AR@AN': + self.ground_truth = {} + + def process(self, data_batch: Sequence[Tuple[Any, dict]], + predictions: Sequence[dict]) -> None: + """Process one batch of data samples and predictions. The processed + results should be stored in ``self.results``, which will be used to + compute the metrics when all batches have been processed. + + Args: + data_batch (Sequence[Tuple[Any, dict]]): A batch of data + from the dataloader. + predictions (Sequence[dict]): A batch of outputs from + the model. + """ + return + + def compute_metrics(self, results: list) -> dict: + """Compute the metrics from processed results. + + If `metric_type` is 'TEM', only dump middle results and do not compute + any metrics. + Args: + results (list): The processed results of each batch. + Returns: + dict: The computed metrics. The keys are the names of the metrics, + and the values are corresponding results. + """ + self.dump_results(results) + if self.metric_type == 'AR@AN': + return self.compute_ARAN(results) + return OrderedDict() + + def compute_ARAN(self, results: list) -> dict: + """AR@AN evaluation metric.""" + eval_results = OrderedDict() + eval_results['auc'] = 0 + eval_results['AR@1'] = 0 + eval_results['AR@5'] = 0 + eval_results['AR@10'] = 0 + eval_results['AR@100'] = 0 + return eval_results + + def dump_results(self, results, version='VERSION 1.3'): + """Save middle or final results to disk.""" + if self.output_format == 'json': + result_dict = self.proposals2json(results) + output_dict = { + 'version': version, + 'results': result_dict, + 'external_data': {} + } + mmengine.dump(output_dict, self.out) + elif self.output_format == 'csv': + os.makedirs(self.out, exist_ok=True) + header = 'action,start,end,tmin,tmax' + for result in results: + video_name, outputs = result + output_path = osp.join(self.out, video_name + '.csv') + np.savetxt( + output_path, + outputs, + header=header, + delimiter=',', + comments='') + else: + raise ValueError( + f'The output format {self.output_format} is not supported.') + + @staticmethod + def proposals2json(results, show_progress=False): + """Convert all proposals to a final dict(json) format. + Args: + results (list[dict]): All proposals. + show_progress (bool): Whether to show the progress bar. + Defaults: False. + Returns: + dict: The final result dict. E.g. + .. code-block:: Python + dict(video-1=[dict(segment=[1.1,2.0]. score=0.9), + dict(segment=[50.1, 129.3], score=0.6)]) + """ + result_dict = {} + print('Convert proposals to json format') + if show_progress: + prog_bar = mmcv.ProgressBar(len(results)) + for result in results: + video_name = result['video_name'] + result_dict[video_name[2:]] = result['proposal_list'] + if show_progress: + prog_bar.update() + return result_dict + + @staticmethod + def _import_proposals(results): + """Read predictions from results.""" + proposals = {} + num_proposals = 0 + for result in results: + video_id = result['video_name'][2:] + this_video_proposals = [] + for proposal in result['proposal_list']: + t_start, t_end = proposal['segment'] + score = proposal['score'] + this_video_proposals.append([t_start, t_end, score]) + num_proposals += 1 + proposals[video_id] = np.array(this_video_proposals) + return proposals, num_proposals diff --git a/aimet_zoo_torch/mmaction2/evaluators/mmaction2_quanteval.py b/aimet_zoo_torch/mmaction2/evaluators/mmaction2_quanteval.py new file mode 100644 index 0000000..c4be472 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/evaluators/mmaction2_quanteval.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +''' AIMET Quantsim evaluation code for mmaction2 BMN model''' + + +import argparse +import torch +from mmengine.config import Config +from mmengine.runner import Runner + +from aimet_torch.model_validator.model_validator import ModelValidator +from aimet_zoo_torch.mmaction2.model.model_definition import MMAction2 + + +def parse_args(): + """Command line argument parser""" + parser = argparse.ArgumentParser("Evaluation script for quantized mmaction2 model") + parser.add_argument('--model-config', help='model configuration to use', required=True, type=str, + default='bmn_w8a8', choices=['bmn_w8a8']) + parser.add_argument('--use-cuda', help='Use GPU for evaluation', action="store_true") + + args = parser.parse_args() + return args + + +def bmn_quanteval(raw_args=None): + """ + quantization evaluation function for BMN model + + :param raw_args: command line arguments (optional) + :return: a dictionary of fp32 and quantized model metrics + """ + args = raw_args if raw_args else parse_args() + device = 'cpu' + if args.use_cuda: + if torch.cuda.is_available(): + device = 'cuda' + else: + raise RuntimeError('Trying to use cuda device while no available cuda device is found!') + + model_downloader = MMAction2(model_config=args.model_config, device=device) + cfg = Config.fromfile(model_downloader.cfg["evaluation"]["config"]) + cfg.load_from = model_downloader.cfg["artifacts"]["url_pre_opt_weights"] + + # build the runner from config + runner = Runner.from_cfg(cfg) + runner.test_evaluator.metrics[0].process = runner.test_evaluator.metrics[0].process_original + runner.test_evaluator.metrics[0].compute_ARAN = runner.test_evaluator.metrics[0].compute_ARAN_original + + # FP32 model testing phase + fp32_metrics = runner.test() + + # quantized model testing phase + dummy_input = torch.randn(model_downloader.input_shape).to(device) + ModelValidator.validate_model(runner.model, model_input=dummy_input) + + runner.test_evaluator.metrics[0].process = runner.test_evaluator.metrics[0].process_dummy + runner.test_evaluator.metrics[0].compute_ARAN = runner.test_evaluator.metrics[0].compute_ARAN_dummy + + model_downloader.model = runner.model + quantsim = model_downloader.get_quantsim(quantized=True) + + runner.model = quantsim.model + runner.test_evaluator.metrics[0].process = runner.test_evaluator.metrics[0].process_original + runner.test_evaluator.metrics[0].compute_ARAN = runner.test_evaluator.metrics[0].compute_ARAN_original + + quantized_metrics = runner.test() + + print(f"FP32 model metrics are {fp32_metrics}") + print(f'quantized model metrics are {quantized_metrics}') + + return {'fp32_metrics': fp32_metrics, + 'quantized_metrics': quantized_metrics} + + +if __name__ == '__main__': + bmn_quanteval() diff --git a/aimet_zoo_torch/mmaction2/mmaction2.md b/aimet_zoo_torch/mmaction2/mmaction2.md new file mode 100644 index 0000000..2eb581f --- /dev/null +++ b/aimet_zoo_torch/mmaction2/mmaction2.md @@ -0,0 +1,56 @@ +# PyTorch mmaction2 + +## Environment Setup + +### Setup AI Model Efficiency Toolkit (AIMET) +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.25/packaging/install.md) before proceeding further. +This model was tested with the `torch_gpu` variant of AIMET 1.25. + + +### Install dependencies +```bash +python -m pip install gdown +``` + +Please follow the steps from [open-mmlab/mmaction2 install guide](https://github.com/open-mmlab/mmaction2#%EF%B8%8F-installation-) +to install mmaction2 as dependency. The package versions we used for open-mmlab are: +- mmaction2 1.0.0 +- mmengine 0.7.3 +- mmcv 2.0.0 + +Append the repo location to your `PYTHONPATH` with the following: +```bash +export PYTHONPATH=$PYTHONPATH: +``` + +### Dataset +Instructions to prepare ActivityNet can be found at: +- https://github.com/open-mmlab/mmaction2/blob/main/tools/data/activitynet/README.md +Note that option 1 was used for this model + +After downloading and processing the dataset, please change the data path to point to your download location in +aimet_zoo_torch/mmaction2/model/configs/localization/bmn/bmn_2xb8-400x100-9e_activitynet-feature.py + +--- + +## Usage +Before running the evaluation script, set your config path in the model cards via replacing with your own path in the +"config" field. The model cards are .json files located under model/model_cards/ + +To run evaluation with QuantSim in AIMET, use the following +```bash +python aimet_zoo_torch/mmaction2/evaluators/mmaction2_quanteval.py --model-config --use-cuda +``` + +Available model configurations are: +- bmn_w8a8 + + +--- + +## Quantization Configuration +- Weight quantization: 8 bits, per tensor symmetric quantization +- Bias parameters are not quantized +- Activation quantization: 8 bits, asymmetric quantization +- Model inputs are quantized +- TF enhanced was used as quantization scheme diff --git a/aimet_zoo_torch/mmaction2/model/__init__.py b/aimet_zoo_torch/mmaction2/model/__init__.py new file mode 100644 index 0000000..1f6c04f --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +from .bmn import * diff --git a/aimet_zoo_torch/mmaction2/model/base_model/__init__.py b/aimet_zoo_torch/mmaction2/model/base_model/__init__.py new file mode 100644 index 0000000..3110a2e --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/base_model/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +from .base_model import BaseModel diff --git a/aimet_zoo_torch/mmaction2/model/base_model/base_model.py b/aimet_zoo_torch/mmaction2/model/base_model/base_model.py new file mode 100644 index 0000000..88a80f7 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/base_model/base_model.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# pylint: skip-file +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +# Copyright (c) OpenMMLab. All rights reserved. +from abc import abstractmethod +from collections import OrderedDict +from typing import Dict, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from mmengine.optim import OptimWrapper +from mmengine.registry import MODELS +from mmengine.utils import is_list_of +from mmengine.model.base_module import BaseModule +from mmengine.model.base_model.data_preprocessor import BaseDataPreprocessor + + +class BaseModel(BaseModule): + """Base class for all algorithmic models. + + BaseModel implements the basic functions of the algorithmic model, such as + weights initialize, batch inputs preprocess(see more information in + :class:`BaseDataPreprocessor`), parse losses, and update model parameters. + + Subclasses inherit from BaseModel only need to implement the forward + method, which implements the logic to calculate loss and predictions, + then can be trained in the runner. + + Examples: + >>> @MODELS.register_module() + >>> class ToyModel(BaseModel): + >>> + >>> def __init__(self): + >>> super().__init__() + >>> self.backbone = nn.Sequential() + >>> self.backbone.add_module('conv1', nn.Conv2d(3, 6, 5)) + >>> self.backbone.add_module('pool', nn.MaxPool2d(2, 2)) + >>> self.backbone.add_module('conv2', nn.Conv2d(6, 16, 5)) + >>> self.backbone.add_module('fc1', nn.Linear(16 * 5 * 5, 120)) + >>> self.backbone.add_module('fc2', nn.Linear(120, 84)) + >>> self.backbone.add_module('fc3', nn.Linear(84, 10)) + >>> + >>> self.criterion = nn.CrossEntropyLoss() + >>> + >>> def forward(self, batch_inputs, data_samples, mode='tensor'): + >>> data_samples = torch.stack(data_samples) + >>> if mode == 'tensor': + >>> return self.backbone(batch_inputs) + >>> elif mode == 'predict': + >>> feats = self.backbone(batch_inputs) + >>> predictions = torch.argmax(feats, 1) + >>> return predictions + >>> elif mode == 'loss': + >>> feats = self.backbone(batch_inputs) + >>> loss = self.criterion(feats, data_samples) + >>> return dict(loss=loss) + + Args: + data_preprocessor (dict, optional): The pre-process config of + :class:`BaseDataPreprocessor`. + init_cfg (dict, optional): The weight initialized config for + :class:`BaseModule`. + + Attributes: + data_preprocessor (:obj:`BaseDataPreprocessor`): Used for + pre-processing data sampled by dataloader to the format accepted by + :meth:`forward`. + init_cfg (dict, optional): Initialization config dict. + """ + + def __init__(self, + data_preprocessor: Optional[Union[dict, nn.Module]] = None, + init_cfg: Optional[dict] = None): + super().__init__(init_cfg) + if data_preprocessor is None: + data_preprocessor = dict(type='BaseDataPreprocessor') + if isinstance(data_preprocessor, nn.Module): + self.data_preprocessor = data_preprocessor + elif isinstance(data_preprocessor, dict): + self.data_preprocessor = MODELS.build(data_preprocessor) + else: + raise TypeError('data_preprocessor should be a `dict` or ' + f'`nn.Module` instance, but got ' + f'{type(data_preprocessor)}') + + def train_step(self, data: Union[dict, tuple, list], + optim_wrapper: OptimWrapper) -> Dict[str, torch.Tensor]: + """Implements the default model training process including + preprocessing, model forward propagation, loss calculation, + optimization, and back-propagation. + + During non-distributed training. If subclasses do not override the + :meth:`train_step`, :class:`EpochBasedTrainLoop` or + :class:`IterBasedTrainLoop` will call this method to update model + parameters. The default parameter update process is as follows: + + 1. Calls ``self.data_processor(data, training=False)`` to collect + batch_inputs and corresponding data_samples(labels). + 2. Calls ``self(batch_inputs, data_samples, mode='loss')`` to get raw + loss + 3. Calls ``self.parse_losses`` to get ``parsed_losses`` tensor used to + backward and dict of loss tensor used to log messages. + 4. Calls ``optim_wrapper.update_params(loss)`` to update model. + + Args: + data (dict or tuple or list): Data sampled from dataset. + optim_wrapper (OptimWrapper): OptimWrapper instance + used to update model parameters. + + Returns: + Dict[str, torch.Tensor]: A ``dict`` of tensor for logging. + """ + # Enable automatic mixed precision training context. + with optim_wrapper.optim_context(self): + data = self.data_preprocessor(data, True) + losses = self._run_forward(data, mode='loss') # type: ignore + parsed_losses, log_vars = self.parse_losses(losses) # type: ignore + optim_wrapper.update_params(parsed_losses) + return log_vars + + def val_step(self, data: Union[tuple, dict, list]) -> list: + """Gets the predictions of given data. + + Calls ``self.data_preprocessor(data, False)`` and + ``self(inputs, data_sample, mode='predict')`` in order. Return the + predictions which will be passed to evaluator. + + Args: + data (dict or tuple or list): Data sampled from dataset. + + Returns: + list: The predictions of given data. + """ + data = self.data_preprocessor(data, False) + return self._run_forward(data, mode='predict') # type: ignore + + def test_step(self, data: Union[dict, tuple, list]) -> list: + """``BaseModel`` implements ``test_step`` the same as ``val_step``. + + Args: + data (dict or tuple or list): Data sampled from dataset. + + Returns: + list: The predictions of given data. + """ + return self._run_forward(data) # type: ignore + + def parse_losses( + self, losses: Dict[str, torch.Tensor] + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """Parses the raw outputs (losses) of the network. + + Args: + losses (dict): Raw output of the network, which usually contain + losses and other necessary information. + + Returns: + tuple[Tensor, dict]: There are two elements. The first is the + loss tensor passed to optim_wrapper which may be a weighted sum + of all losses, and the second is log_vars which will be sent to + the logger. + """ + log_vars = [] + for loss_name, loss_value in losses.items(): + if isinstance(loss_value, torch.Tensor): + log_vars.append([loss_name, loss_value.mean()]) + elif is_list_of(loss_value, torch.Tensor): + log_vars.append( + [loss_name, + sum(_loss.mean() for _loss in loss_value)]) + else: + raise TypeError( + f'{loss_name} is not a tensor or list of tensors') + + loss = sum(value for key, value in log_vars if 'loss' in key) + log_vars.insert(0, ['loss', loss]) + log_vars = OrderedDict(log_vars) # type: ignore + + return loss, log_vars # type: ignore + + def to(self, *args, **kwargs) -> nn.Module: + """Overrides this method to call :meth:`BaseDataPreprocessor.to` + additionally. + + Returns: + nn.Module: The model itself. + """ + + # Since Torch has not officially merged + # the npu-related fields, using the _parse_to function + # directly will cause the NPU to not be found. + # Here, the input parameters are processed to avoid errors. + if args and isinstance(args[0], str) and 'npu' in args[0]: + args = tuple( + [list(args)[0].replace('npu', torch.npu.native_device)]) + if kwargs and 'npu' in str(kwargs.get('device', '')): + kwargs['device'] = kwargs['device'].replace( + 'npu', torch.npu.native_device) + + device = torch._C._nn._parse_to(*args, **kwargs)[0] + if device is not None: + self._set_device(torch.device(device)) + return super().to(*args, **kwargs) + + def cuda( + self, + device: Optional[Union[int, str, torch.device]] = None, + ) -> nn.Module: + """Overrides this method to call :meth:`BaseDataPreprocessor.cuda` + additionally. + + Returns: + nn.Module: The model itself. + """ + if device is None or isinstance(device, int): + device = torch.device('cuda', index=device) + self._set_device(torch.device(device)) + return super().cuda(device) + + def mlu( + self, + device: Union[int, str, torch.device, None] = None, + ) -> nn.Module: + """Overrides this method to call :meth:`BaseDataPreprocessor.mlu` + additionally. + + Returns: + nn.Module: The model itself. + """ + device = torch.device('mlu', torch.mlu.current_device()) + self._set_device(device) + return super().mlu() + + def npu( + self, + device: Union[int, str, torch.device, None] = None, + ) -> nn.Module: + """Overrides this method to call :meth:`BaseDataPreprocessor.npu` + additionally. + + Returns: + nn.Module: The model itself. + + Note: + This generation of NPU(Ascend910) does not support + the use of multiple cards in a single process, + so the index here needs to be consistent with the default device + """ + device = torch.npu.current_device() + self._set_device(device) + return super().npu() + + def cpu(self, *args, **kwargs) -> nn.Module: + """Overrides this method to call :meth:`BaseDataPreprocessor.cpu` + additionally. + + Returns: + nn.Module: The model itself. + """ + self._set_device(torch.device('cpu')) + return super().cpu() + + def _set_device(self, device: torch.device) -> None: + """Recursively set device for `BaseDataPreprocessor` instance. + + Args: + device (torch.device): the desired device of the parameters and + buffers in this module. + """ + + def apply_fn(module): + if not isinstance(module, BaseDataPreprocessor): + return + if device is not None: + module._device = device + + self.apply(apply_fn) + + @abstractmethod + def forward(self, + inputs: torch.Tensor, + data_samples: Optional[list] = None, + mode: str = 'tensor') -> Union[Dict[str, torch.Tensor], list]: + """Returns losses or predictions of training, validation, testing, and + simple inference process. + + ``forward`` method of BaseModel is an abstract method, its subclasses + must implement this method. + + Accepts ``batch_inputs`` and ``data_sample`` processed by + :attr:`data_preprocessor`, and returns results according to mode + arguments. + + During non-distributed training, validation, and testing process, + ``forward`` will be called by ``BaseModel.train_step``, + ``BaseModel.val_step`` and ``BaseModel.val_step`` directly. + + During distributed data parallel training process, + ``MMSeparateDistributedDataParallel.train_step`` will first call + ``DistributedDataParallel.forward`` to enable automatic + gradient synchronization, and then call ``forward`` to get training + loss. + + Args: + inputs (torch.Tensor): batch input tensor collated by + :attr:`data_preprocessor`. + data_samples (list, optional): + data samples collated by :attr:`data_preprocessor`. + mode (str): mode should be one of ``loss``, ``predict`` and + ``tensor`` + + - ``loss``: Called by ``train_step`` and return loss ``dict`` + used for logging + - ``predict``: Called by ``val_step`` and ``test_step`` + and return list of results used for computing metric. + - ``tensor``: Called by custom use to get ``Tensor`` type + results. + + Returns: + dict or list: + - If ``mode == loss``, return a ``dict`` of loss tensor used + for backward and logging. + - If ``mode == predict``, return a ``list`` of inference + results. + - If ``mode == tensor``, return a tensor or ``tuple`` of tensor + or ``dict`` of tensor for custom use. + """ + + def _run_forward(self, data: Union[dict, tuple, list]): + """Unpacks data for :meth:`forward` + + Args: + data (dict or tuple or list): Data sampled from dataset. + mode (str): Mode of forward. + + Returns: + dict or list: Results of training or testing mode. + """ + if isinstance(data, dict): + results = self(**data) + elif isinstance(data, (list, tuple)): + results = self(*data) + else: + raise TypeError('Output of `data_preprocessor` should be ' + f'list, tuple or dict, but got {type(data)}') + return results diff --git a/aimet_zoo_torch/mmaction2/model/bmn.py b/aimet_zoo_torch/mmaction2/model/bmn.py new file mode 100644 index 0000000..e76a176 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/bmn.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# pylint: skip-file +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +# Copyright (c) OpenMMLab. All rights reserved. +import math + +import numpy as np +import torch +import torch.nn as nn +from .base_model import BaseModel + +from mmaction.registry import MODELS +from mmaction.models.localizers.utils import post_processing, temporal_iop, temporal_iou + +from aimet_torch.elementwise_ops import MatMul + + +@MODELS.register_module() +class BMN_AIMET(BaseModel): + """Boundary Matching Network for temporal action proposal generation. + + Please refer `BMN: Boundary-Matching Network for Temporal Action Proposal + Generation `_. + Code Reference https://github.com/JJBOY/BMN-Boundary-Matching-Network + Args: + temporal_dim (int): Total frames selected for each video. + boundary_ratio (float): Ratio for determining video boundaries. + num_samples (int): Number of samples for each proposal. + num_samples_per_bin (int): Number of bin samples for each sample. + feat_dim (int): Feature dimension. + soft_nms_alpha (float): Soft NMS alpha. + soft_nms_low_threshold (float): Soft NMS low threshold. + soft_nms_high_threshold (float): Soft NMS high threshold. + post_process_top_k (int): Top k proposals in post process. + feature_extraction_interval (int): + Interval used in feature extraction. Default: 16. + loss_cls (dict): Config for building loss. + Default: ``dict(type='BMNLoss')``. + hidden_dim_1d (int): Hidden dim for 1d conv. Default: 256. + hidden_dim_2d (int): Hidden dim for 2d conv. Default: 128. + hidden_dim_3d (int): Hidden dim for 3d conv. Default: 512. + """ + + def __init__(self, + temporal_dim, + boundary_ratio, + num_samples, + num_samples_per_bin, + feat_dim, + soft_nms_alpha, + soft_nms_low_threshold, + soft_nms_high_threshold, + post_process_top_k, + feature_extraction_interval=16, + loss_cls=dict(type='BMNLoss'), + hidden_dim_1d=256, + hidden_dim_2d=128, + hidden_dim_3d=512): + super().__init__() + + self.tscale = temporal_dim + self.boundary_ratio = boundary_ratio + self.num_samples = num_samples + self.num_samples_per_bin = num_samples_per_bin + self.feat_dim = feat_dim + self.soft_nms_alpha = soft_nms_alpha + self.soft_nms_low_threshold = soft_nms_low_threshold + self.soft_nms_high_threshold = soft_nms_high_threshold + self.post_process_top_k = post_process_top_k + self.feature_extraction_interval = feature_extraction_interval + self.loss_cls = MODELS.build(loss_cls) + self.hidden_dim_1d = hidden_dim_1d + self.hidden_dim_2d = hidden_dim_2d + self.hidden_dim_3d = hidden_dim_3d + + self._get_interp1d_mask() + + # Base Module + self.x_1d_b = nn.Sequential( + nn.Conv1d( + self.feat_dim, + self.hidden_dim_1d, + kernel_size=3, + padding=1, + groups=4), nn.ReLU(inplace=True), + nn.Conv1d( + self.hidden_dim_1d, + self.hidden_dim_1d, + kernel_size=3, + padding=1, + groups=4), nn.ReLU(inplace=True)) + + # Temporal Evaluation Module + self.x_1d_s = nn.Sequential( + nn.Conv1d( + self.hidden_dim_1d, + self.hidden_dim_1d, + kernel_size=3, + padding=1, + groups=4), nn.ReLU(inplace=True), + nn.Conv1d(self.hidden_dim_1d, 1, kernel_size=1), nn.Sigmoid()) + self.x_1d_e = nn.Sequential( + nn.Conv1d( + self.hidden_dim_1d, + self.hidden_dim_1d, + kernel_size=3, + padding=1, + groups=4), nn.ReLU(inplace=True), + nn.Conv1d(self.hidden_dim_1d, 1, kernel_size=1), nn.Sigmoid()) + + # Proposal Evaluation Module + self.x_1d_p = nn.Sequential( + nn.Conv1d( + self.hidden_dim_1d, + self.hidden_dim_1d, + kernel_size=3, + padding=1), nn.ReLU(inplace=True)) + self.x_3d_p = nn.Sequential( + nn.Conv3d( + self.hidden_dim_1d, + self.hidden_dim_3d, + kernel_size=(self.num_samples, 1, 1)), nn.ReLU(inplace=True)) + self.x_2d_p = nn.Sequential( + nn.Conv2d(self.hidden_dim_3d, self.hidden_dim_2d, kernel_size=1), + nn.ReLU(inplace=True), + nn.Conv2d( + self.hidden_dim_2d, + self.hidden_dim_2d, + kernel_size=3, + padding=1), nn.ReLU(inplace=True), + nn.Conv2d( + self.hidden_dim_2d, + self.hidden_dim_2d, + kernel_size=3, + padding=1), nn.ReLU(inplace=True), + nn.Conv2d(self.hidden_dim_2d, 2, kernel_size=1), nn.Sigmoid()) + self.matmul = MatMul() + self.anchors_tmins, self.anchors_tmaxs = self._temporal_anchors( + -0.5, 1.5) + self.match_map = self._match_map() + # self.bm_mask = self._get_bm_mask() + self.register_buffer('bm_mask', self._get_bm_mask()) + + def init_weights(self) -> None: + """Initiate the parameters from scratch.""" + pass + + def forward(self, inputs, **kwargs): + """The unified entry for a forward process in both training and test. + + Note that this method doesn't handle neither back propagation nor + optimizer updating, which are done in the :meth:`train_step`. + + Args: + inputs (Tensor): The input tensor with shape + (N, C, ...) in general. + + Returns: + - If return a tensor or a tuple of tensor. + """ + if (inputs.dim() == 2): + inputs = torch.stack((inputs, )) + return self._forward(inputs) + + def loss(self, batch_inputs, batch_data_samples, **kwargs): + """Calculate losses from a batch of inputs and data samples. + + Args: + batch_inputs (Tensor): Raw Inputs of the recognizer. + These should usually be mean centered and std scaled. + batch_data_samples (List[:obj:`ActionDataSample`]): The batch + data samples. It usually includes information such + as ``gt_labels``. + + Returns: + dict: A dictionary of loss components. + """ + gt_bbox = [ + sample.gt_instances['gt_bbox'] for sample in batch_data_samples + ] + label_confidence, label_start, label_end = self.generate_labels( + gt_bbox) + + device = batch_inputs.device + label_confidence = label_confidence.to(device) + label_start = label_start.to(device) + label_end = label_end.to(device) + + confidence_map, start, end = self._forward(batch_inputs) + + loss = self.loss_cls(confidence_map, start, end, label_confidence, + label_start, label_end, self.bm_mask) + loss_dict = dict(loss=loss[0]) + return loss_dict + + def predict(self, confidence_map, start, end, batch_data_samples, **kwargs): + """Define the computation performed at every call when testing.""" + start_scores = start[0].cpu().numpy() + end_scores = end[0].cpu().numpy() + cls_confidence = (confidence_map[0][1]).cpu().numpy() + reg_confidence = (confidence_map[0][0]).cpu().numpy() + + max_start = max(start_scores) + max_end = max(end_scores) + + # generate the set of start points and end points + start_bins = np.zeros(len(start_scores)) + start_bins[0] = 1 # [1,0,0...,0,0] + end_bins = np.zeros(len(end_scores)) + end_bins[-1] = 1 # [0,0,0...,0,1] + for idx in range(1, self.tscale - 1): + if start_scores[idx] > start_scores[ + idx + 1] and start_scores[idx] > start_scores[idx - 1]: + start_bins[idx] = 1 + elif start_scores[idx] > (0.5 * max_start): + start_bins[idx] = 1 + if end_scores[idx] > end_scores[ + idx + 1] and end_scores[idx] > end_scores[idx - 1]: + end_bins[idx] = 1 + elif end_scores[idx] > (0.5 * max_end): + end_bins[idx] = 1 + + # iterate through all combinations of start_index and end_index + new_proposals = [] + for idx in range(self.tscale): + for jdx in range(self.tscale): + start_index = jdx + end_index = start_index + idx + 1 + if end_index < self.tscale and start_bins[ + start_index] == 1 and end_bins[end_index] == 1: + tmin = start_index / self.tscale + tmax = end_index / self.tscale + tmin_score = start_scores[start_index] + tmax_score = end_scores[end_index] + cls_score = cls_confidence[idx, jdx] + reg_score = reg_confidence[idx, jdx] + score = tmin_score * tmax_score * cls_score * reg_score + new_proposals.append([ + tmin, tmax, tmin_score, tmax_score, cls_score, + reg_score, score + ]) + new_proposals = np.stack(new_proposals) + video_info = batch_data_samples[0].metainfo + proposal_list = post_processing(new_proposals, video_info, + self.soft_nms_alpha, + self.soft_nms_low_threshold, + self.soft_nms_high_threshold, + self.post_process_top_k, + self.feature_extraction_interval) + output = [ + dict( + video_name=video_info['video_name'], + proposal_list=proposal_list) + ] + return output + + @staticmethod + def _get_interp1d_bin_mask(seg_tmin, seg_tmax, tscale, num_samples, + num_samples_per_bin): + """Generate sample mask for a boundary-matching pair.""" + plen = float(seg_tmax - seg_tmin) + plen_sample = plen / (num_samples * num_samples_per_bin - 1.0) + total_samples = [ + seg_tmin + plen_sample * i + for i in range(num_samples * num_samples_per_bin) + ] + p_mask = [] + for idx in range(num_samples): + bin_samples = total_samples[idx * num_samples_per_bin:(idx + 1) * + num_samples_per_bin] + bin_vector = np.zeros(tscale) + for sample in bin_samples: + sample_upper = math.ceil(sample) + sample_decimal, sample_down = math.modf(sample) + if 0 <= int(sample_down) <= (tscale - 1): + bin_vector[int(sample_down)] += 1 - sample_decimal + if 0 <= int(sample_upper) <= (tscale - 1): + bin_vector[int(sample_upper)] += sample_decimal + bin_vector = 1.0 / num_samples_per_bin * bin_vector + p_mask.append(bin_vector) + p_mask = np.stack(p_mask, axis=1) + return p_mask + + def _get_interp1d_mask(self): + """Generate sample mask for each point in Boundary-Matching Map.""" + mask_mat = [] + for start_index in range(self.tscale): + mask_mat_vector = [] + for duration_index in range(self.tscale): + if start_index + duration_index < self.tscale: + p_tmin = start_index + p_tmax = start_index + duration_index + center_len = float(p_tmax - p_tmin) + 1 + sample_tmin = p_tmin - (center_len * self.boundary_ratio) + sample_tmax = p_tmax + (center_len * self.boundary_ratio) + p_mask = self._get_interp1d_bin_mask( + sample_tmin, sample_tmax, self.tscale, + self.num_samples, self.num_samples_per_bin) + else: + p_mask = np.zeros([self.tscale, self.num_samples]) + mask_mat_vector.append(p_mask) + mask_mat_vector = np.stack(mask_mat_vector, axis=2) + mask_mat.append(mask_mat_vector) + mask_mat = np.stack(mask_mat, axis=3) + mask_mat = mask_mat.astype(np.float32) + self.sample_mask = nn.Parameter( + torch.tensor(mask_mat).view(self.tscale, -1), requires_grad=False) + + def _get_bm_mask(self): + """Generate Boundary-Matching Mask.""" + bm_mask = [] + for idx in range(self.tscale): + mask_vector = [1] * (self.tscale - idx) + [0] * idx + bm_mask.append(mask_vector) + bm_mask = torch.tensor(bm_mask, dtype=torch.float) + return bm_mask + + def _match_map(self): + """Generate match map.""" + temporal_gap = 1. / self.tscale + match_map = [] + for idx in range(self.tscale): + match_window = [] + tmin = temporal_gap * idx + for jdx in range(1, self.tscale + 1): + tmax = tmin + temporal_gap * jdx + match_window.append([tmin, tmax]) + match_map.append(match_window) + match_map = np.array(match_map) + match_map = np.transpose(match_map, [1, 0, 2]) + match_map = np.reshape(match_map, [-1, 2]) + return match_map + + def _temporal_anchors(self, tmin_offset=0., tmax_offset=1.): + """Generate temporal anchors. + + Args: + tmin_offset (int): Offset for the minimum value of temporal anchor. + Default: 0. + tmax_offset (int): Offset for the maximum value of temporal anchor. + Default: 1. + Returns: + tuple[Sequence[float]]: The minimum and maximum values of temporal + anchors. + """ + temporal_gap = 1. / self.tscale + anchors_tmins = [] + anchors_tmaxs = [] + for i in range(self.tscale): + anchors_tmins.append(temporal_gap * (i + tmin_offset)) + anchors_tmaxs.append(temporal_gap * (i + tmax_offset)) + + return anchors_tmins, anchors_tmaxs + + def _forward(self, x): + """Define the computation performed at every call. + + Args: + x (torch.Tensor): The input data. + Returns: + torch.Tensor: The output of the module. + """ + # x.shape [batch_size, self.feat_dim, self.tscale] + base_feature = self.x_1d_b(x) + # base_feature.shape [batch_size, self.hidden_dim_1d, self.tscale] + start = self.x_1d_s(base_feature).squeeze(1) + # start.shape [batch_size, self.tscale] + end = self.x_1d_e(base_feature).squeeze(1) + # end.shape [batch_size, self.tscale] + confidence_map = self.x_1d_p(base_feature) + # [batch_size, self.hidden_dim_1d, self.tscale] + confidence_map = self._boundary_matching_layer(confidence_map) + # [batch_size, self.hidden_dim_1d,, self.num_sampls, self.tscale, self.tscale] # noqa + confidence_map = self.x_3d_p(confidence_map).squeeze(2) + # [batch_size, self.hidden_dim_3d, self.tscale, self.tscale] + confidence_map = self.x_2d_p(confidence_map) + # [batch_size, 2, self.tscale, self.tscale] + + return confidence_map, start, end + + def _boundary_matching_layer(self, x): + """Generate matching layer.""" + input_size = x.size() + out = self.matmul(x, + self.sample_mask).reshape(input_size[0], + input_size[1], + self.num_samples, + self.tscale, self.tscale) + return out + + def generate_labels(self, gt_bbox): + """Generate training labels.""" + # TODO: do this without numpy + match_score_confidence_list = [] + match_score_start_list = [] + match_score_end_list = [] + for every_gt_bbox in gt_bbox: + gt_iou_map = [] + every_gt_bbox = every_gt_bbox.cpu() + for start, end in every_gt_bbox: + if isinstance(start, torch.Tensor): + start = start.numpy() + if isinstance(end, torch.Tensor): + end = end.numpy() + current_gt_iou_map = temporal_iou(self.match_map[:, 0], + self.match_map[:, 1], start, + end) + current_gt_iou_map = np.reshape(current_gt_iou_map, + [self.tscale, self.tscale]) + gt_iou_map.append(current_gt_iou_map) + gt_iou_map = np.array(gt_iou_map).astype(np.float32) + gt_iou_map = np.max(gt_iou_map, axis=0) + + gt_tmins = every_gt_bbox[:, 0] + gt_tmaxs = every_gt_bbox[:, 1] + + gt_len_pad = 3 * (1. / self.tscale) + + gt_start_bboxs = np.stack( + (gt_tmins - gt_len_pad / 2, gt_tmins + gt_len_pad / 2), axis=1) + gt_end_bboxs = np.stack( + (gt_tmaxs - gt_len_pad / 2, gt_tmaxs + gt_len_pad / 2), axis=1) + + match_score_start = [] + match_score_end = [] + + for anchor_tmin, anchor_tmax in zip(self.anchors_tmins, + self.anchors_tmaxs): + match_score_start.append( + np.max( + temporal_iop(anchor_tmin, anchor_tmax, + gt_start_bboxs[:, 0], gt_start_bboxs[:, + 1]))) + match_score_end.append( + np.max( + temporal_iop(anchor_tmin, anchor_tmax, + gt_end_bboxs[:, 0], gt_end_bboxs[:, 1]))) + match_score_confidence_list.append(gt_iou_map) + match_score_start_list.append(match_score_start) + match_score_end_list.append(match_score_end) + + def to_tensor(x): + return torch.Tensor(np.array(x)) + + match_score_confidence_list = to_tensor(match_score_confidence_list) + match_score_start_list = to_tensor(match_score_start_list) + match_score_end_list = to_tensor(match_score_end_list) + return (match_score_confidence_list, match_score_start_list, + match_score_end_list) diff --git a/aimet_zoo_torch/mmaction2/model/configs/__init__.py b/aimet_zoo_torch/mmaction2/model/configs/__init__.py new file mode 100644 index 0000000..01720c8 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/configs/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= diff --git a/aimet_zoo_torch/mmaction2/model/configs/_base_/__init__.py b/aimet_zoo_torch/mmaction2/model/configs/_base_/__init__.py new file mode 100644 index 0000000..01720c8 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/configs/_base_/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= diff --git a/aimet_zoo_torch/mmaction2/model/configs/_base_/default_runtime.py b/aimet_zoo_torch/mmaction2/model/configs/_base_/default_runtime.py new file mode 100644 index 0000000..c32a323 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/configs/_base_/default_runtime.py @@ -0,0 +1,25 @@ +# pylint: skip-file +default_scope = 'mmaction' + +default_hooks = dict( + runtime_info=dict(type='RuntimeInfoHook'), + timer=dict(type='IterTimerHook'), + logger=dict(type='LoggerHook', interval=20, ignore_last=False), + param_scheduler=dict(type='ParamSchedulerHook'), + checkpoint=dict(type='CheckpointHook', interval=1, save_best='auto'), + sampler_seed=dict(type='DistSamplerSeedHook'), + sync_buffers=dict(type='SyncBuffersHook')) + +env_cfg = dict( + cudnn_benchmark=False, + mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0), + dist_cfg=dict(backend='nccl')) + +log_processor = dict(type='LogProcessor', window_size=20, by_epoch=True) + +vis_backends = [dict(type='LocalVisBackend')] +visualizer = dict(type='ActionVisualizer', vis_backends=vis_backends) + +log_level = 'INFO' +load_from = None +resume = False diff --git a/aimet_zoo_torch/mmaction2/model/configs/_base_/models/__init__.py b/aimet_zoo_torch/mmaction2/model/configs/_base_/models/__init__.py new file mode 100644 index 0000000..01720c8 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/configs/_base_/models/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= diff --git a/aimet_zoo_torch/mmaction2/model/configs/_base_/models/bmn_400x100.py b/aimet_zoo_torch/mmaction2/model/configs/_base_/models/bmn_400x100.py new file mode 100644 index 0000000..4f81580 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/configs/_base_/models/bmn_400x100.py @@ -0,0 +1,13 @@ +# pylint: skip-file +# model settings +model = dict( + type='BMN_AIMET', + temporal_dim=100, + boundary_ratio=0.5, + num_samples=32, + num_samples_per_bin=3, + feat_dim=400, + soft_nms_alpha=0.4, + soft_nms_low_threshold=0.5, + soft_nms_high_threshold=0.9, + post_process_top_k=100) diff --git a/aimet_zoo_torch/mmaction2/model/configs/localization/__init__.py b/aimet_zoo_torch/mmaction2/model/configs/localization/__init__.py new file mode 100644 index 0000000..01720c8 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/configs/localization/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= diff --git a/aimet_zoo_torch/mmaction2/model/configs/localization/bmn/__init__.py b/aimet_zoo_torch/mmaction2/model/configs/localization/bmn/__init__.py new file mode 100644 index 0000000..01720c8 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/configs/localization/bmn/__init__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= diff --git a/aimet_zoo_torch/mmaction2/model/configs/localization/bmn/bmn_2xb8-400x100-9e_activitynet-feature.py b/aimet_zoo_torch/mmaction2/model/configs/localization/bmn/bmn_2xb8-400x100-9e_activitynet-feature.py new file mode 100644 index 0000000..140d991 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/configs/localization/bmn/bmn_2xb8-400x100-9e_activitynet-feature.py @@ -0,0 +1,109 @@ +# pylint: skip-file +_base_ = [ + '../../_base_/models/bmn_400x100.py', '../../_base_/default_runtime.py' +] + +# dataset settings +dataset_type = 'ActivityNetDataset' +data_root = '/path/to/data/ActivityNet/activitynet_feature_cuhk/csv_mean_100/' +data_root_val = '/path/to/data/data/ActivityNet/activitynet_feature_cuhk/csv_mean_100/' +ann_file_train = '/path/to/data/data/ActivityNet/anet_anno_train.json' +ann_file_val = '/path/to/data/data/ActivityNet/anet_anno_val.json' +ann_file_test = '/path/to/data/data/ActivityNet/anet_anno_val.json' + +train_pipeline = [ + dict(type='LoadLocalizationFeature'), + dict(type='GenerateLocalizationLabels'), + dict( + type='PackLocalizationInputs', + keys=('gt_bbox', ), + meta_keys=('video_name', )) +] + +val_pipeline = [ + dict(type='LoadLocalizationFeature'), + dict(type='GenerateLocalizationLabels'), + dict( + type='PackLocalizationInputs', + keys=('gt_bbox', ), + meta_keys=('video_name', 'duration_second', 'duration_frame', + 'annotations', 'feature_frame')) +] + +test_pipeline = [ + dict(type='LoadLocalizationFeature'), + dict( + type='PackLocalizationInputs', + keys=('gt_bbox', ), + meta_keys=('video_name', 'duration_second', 'duration_frame', + 'annotations', 'feature_frame')) +] + +train_dataloader = dict( + batch_size=8, + num_workers=8, + persistent_workers=True, + sampler=dict(type='DefaultSampler', shuffle=True), + drop_last=True, + dataset=dict( + type=dataset_type, + ann_file=ann_file_train, + data_prefix=dict(video=data_root), + pipeline=train_pipeline)) + +val_dataloader = dict( + batch_size=1, + num_workers=8, + persistent_workers=True, + sampler=dict(type='DefaultSampler', shuffle=False), + dataset=dict( + type=dataset_type, + ann_file=ann_file_val, + data_prefix=dict(video=data_root_val), + pipeline=val_pipeline, + test_mode=True)) + +test_dataloader = dict( + batch_size=1, + num_workers=8, + persistent_workers=True, + sampler=dict(type='DefaultSampler', shuffle=False), + dataset=dict( + type=dataset_type, + ann_file=ann_file_test, + data_prefix=dict(video=data_root_val), + pipeline=test_pipeline, + test_mode=True)) + +max_epochs = 9 +train_cfg = dict( + type='EpochBasedTrainLoop', + max_epochs=max_epochs, + val_begin=1, + val_interval=1) + +val_cfg = dict(type='ValLoop') +test_cfg = dict(type='AIMETTestLoop') + +optim_wrapper = dict( + optimizer=dict(type='Adam', lr=0.001, weight_decay=0.0001), + clip_grad=dict(max_norm=40, norm_type=2)) + +param_scheduler = [ + dict( + type='MultiStepLR', + begin=0, + end=max_epochs, + by_epoch=True, + milestones=[ + 7, + ], + gamma=0.1) +] + +work_dir = './work_dirs/bmn_400x100_2x8_9e_activitynet_feature' +test_evaluator = dict( + type='AIMETANetMetric', + metric_type='AR@AN', + dump_config=dict(out=f'{work_dir}/results.json', output_format='json')) +val_evaluator = test_evaluator diff --git a/aimet_zoo_torch/mmaction2/model/model_cards/bmn_w8a8.json b/aimet_zoo_torch/mmaction2/model/model_cards/bmn_w8a8.json new file mode 100644 index 0000000..de575cc --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/model_cards/bmn_w8a8.json @@ -0,0 +1,26 @@ +{ + "name": "bmn", + "framework": "pytorch", + "task": "action localization", + "input_shape": [null, 400, 100], + "evaluation": { + "config": "/path/to/aimet-model-zoo/aimet_zoo_torch/mmaction2/model/configs/localization/bmn/bmn_2xb8-400x100-9e_activitynet-feature.py" + }, + "optimization_config": { + "quantization_configuration": + { + "param_bw": 8, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "techniques": null + } + }, + "artifacts": { + "url_pre_opt_weights": "https://download.openmmlab.com/mmaction/v1.0/localization/bmn/bmn_2xb8-400x100-9e_activitynet-feature_20220908-79f92857.pth", + "url_post_opt_weights": null, + "url_adaround_encodings": null, + "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/torch_mmaction2/bmn_w8a8_torch.encodings", + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.24/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" + } +} diff --git a/aimet_zoo_torch/mmaction2/model/model_definition.py b/aimet_zoo_torch/mmaction2/model/model_definition.py new file mode 100644 index 0000000..58c70c7 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/model/model_definition.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +"""Model downloader definition for mmaction2 BMN model""" + + +import os +import json +import pathlib + +import torch +from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim + +from aimet_zoo_torch.common.downloader import Downloader + + +class MMAction2(Downloader): + """ + Downloader class for mmaction2 BMN model + """ + def __init__(self, model_config=None, **kwargs): + """ + :param model_config: named model config from which to obtain model artifacts and arguments. + If provided, overwrites the other arguments passed to this object + """ + parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) + self.cfg = False + if model_config: + config_filepath = os.path.join(parent_dir, f'model_cards/{model_config}.json') + try: + with open(config_filepath) as f_in: + self.cfg = json.load(f_in) + except FileNotFoundError: + print(f"Trying to open a model_config file from a non-existent path {config_filepath}!") + raise + if self.cfg: + Downloader.__init__(self, + url_pre_opt_weights = self.cfg['artifacts']['url_pre_opt_weights'], + url_post_opt_weights = self.cfg['artifacts']['url_post_opt_weights'], + url_aimet_encodings = self.cfg['artifacts']['url_aimet_encodings'], + url_aimet_config = self.cfg['artifacts']['url_aimet_config'], + model_dir = parent_dir, + model_config = model_config) + self.input_shape = tuple(x if x is not None else 1 for x in self.cfg['input_shape']) + self.device = kwargs.get('device', 'cuda') + self.model = None + + def from_pretrained(self): + """load pretrained weights""" + if not self.cfg: + raise NotImplementedError('There are no pretrained weights available for the model_config passed') + self._download_post_opt_weights() + self._download_aimet_config() + self._download_aimet_encodings() + self._download_adaround_encodings() + + self.model.eval() + + def get_quantsim(self, quantized=False): + """get quantsim object with pre-loaded encodings""" + if not self.cfg: + raise NotImplementedError('There is no Quantization Simulation available for the model_config passed') + self.from_pretrained() + + dummy_input = torch.rand(self.input_shape, device=self.device) + quant_config = self.cfg['optimization_config']['quantization_configuration'] + kwargs = { + 'quant_scheme': quant_config['quant_scheme'], + 'default_param_bw': quant_config['param_bw'], + 'default_output_bw': quant_config['output_bw'], + 'config_file': self.path_aimet_config, + 'dummy_input': dummy_input} + sim = QuantizationSimModel(self.model, **kwargs) + if self.path_aimet_encodings and quantized: + load_encodings_to_sim(sim, self.path_aimet_encodings) + print('load_encodings_to_sim finished!') + + sim.model.eval() + return sim diff --git a/aimet_zoo_torch/mmaction2/runner/__init__.py b/aimet_zoo_torch/mmaction2/runner/__init__.py new file mode 100644 index 0000000..8a836e4 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/runner/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +from .loops import AIMETTestLoop diff --git a/aimet_zoo_torch/mmaction2/runner/loops.py b/aimet_zoo_torch/mmaction2/runner/loops.py new file mode 100644 index 0000000..ee11ac6 --- /dev/null +++ b/aimet_zoo_torch/mmaction2/runner/loops.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# pylint: skip-file +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +# Copyright (c) OpenMMLab. All rights reserved. +import logging +from typing import Dict, List, Sequence, Union + +import torch +from torch.utils.data import DataLoader + +from mmengine.evaluator import Evaluator +from mmengine.logging import print_log +from mmengine.registry import LOOPS +from mmengine.runner.amp import autocast +from mmengine.runner.base_loop import BaseLoop + + +@LOOPS.register_module() +class AIMETTestLoop(BaseLoop): + """Loop for test. + + Args: + runner (Runner): A reference of runner. + dataloader (Dataloader or dict): A dataloader object or a dict to + build a dataloader. + evaluator (Evaluator or dict or list): Used for computing metrics. + fp16 (bool): Whether to enable fp16 testing. Defaults to + False. + """ + + def __init__(self, + runner, + dataloader: Union[DataLoader, Dict], + evaluator: Union[Evaluator, Dict, List], + fp16: bool = False): + super().__init__(runner, dataloader) + + if isinstance(evaluator, dict) or isinstance(evaluator, list): + self.evaluator = runner.build_evaluator(evaluator) # type: ignore + else: + self.evaluator = evaluator # type: ignore + if hasattr(self.dataloader.dataset, 'metainfo'): + self.evaluator.dataset_meta = self.dataloader.dataset.metainfo + self.runner.visualizer.dataset_meta = \ + self.dataloader.dataset.metainfo + else: + print_log( + f'Dataset {self.dataloader.dataset.__class__.__name__} has no ' + 'metainfo. ``dataset_meta`` in evaluator, metric and ' + 'visualizer will be None.', + logger='current', + level=logging.WARNING) + self.fp16 = fp16 + + def run(self) -> dict: + """Launch test.""" + self.runner.call_hook('before_test') + self.runner.call_hook('before_test_epoch') + self.runner.model.eval() + for idx, data_batch in enumerate(self.dataloader): + self.run_iter(idx, data_batch) + + # compute metrics + metrics = self.evaluator.evaluate(len(self.dataloader.dataset)) + self.runner.call_hook('after_test_epoch', metrics=metrics) + self.runner.call_hook('after_test') + return metrics + + @torch.no_grad() + def run_iter(self, idx, data_batch: Sequence[dict]) -> None: + """Iterate one mini-batch. + + Args: + data_batch (Sequence[dict]): Batch of data from dataloader. + """ + if isinstance(data_batch, dict): + data_batch['inputs'][0] = data_batch['inputs'][0].cuda() + data_batch = [data_batch['inputs'], data_batch['data_samples']] + else: + data_batch = data_batch.cuda() + self.runner.call_hook( + 'before_test_iter', batch_idx=idx, data_batch=data_batch) + # predictions should be sequence of BaseDataElement + with autocast(enabled=self.fp16): + if len(data_batch) > 1: + confidence_map, start, end = self.runner.model.test_step(data_batch[0]) + outputs = self.runner.model.predict(confidence_map, start, end, data_batch[1]) + else: + outputs = self.runner.model.test_step(data_batch) + self.evaluator.process(data_samples=outputs, data_batch=data_batch) + self.runner.call_hook( + 'after_test_iter', + batch_idx=idx, + data_batch=data_batch, + outputs=outputs) diff --git a/aimet_zoo_torch/mobilebert/dataloader/dataloaders.py b/aimet_zoo_torch/mobilebert/dataloader/dataloaders.py index 42bee3e..de40e49 100644 --- a/aimet_zoo_torch/mobilebert/dataloader/dataloaders.py +++ b/aimet_zoo_torch/mobilebert/dataloader/dataloaders.py @@ -303,7 +303,9 @@ def preprocess_function(examples): return datasets -def eval_function(model,tokenizer,datasets,data_args,training_args): +def eval_function(model,tokenizer,datasets,data_args,training_args,max_eval_samples=None): + if max_eval_samples is not None: + data_args.max_eval_samples = max_eval_samples ## case 1. when dataset is glue if hasattr(data_args,'task_name'): train_dataset = datasets["train"] @@ -366,16 +368,25 @@ def compute_metrics(p: EvalPrediction): # Loop to handle MNLI double evaluation (matched, mis-matched) tasks = [data_args.task_name] - eval_datasets = [eval_dataset] + + if data_args.max_eval_samples is not None: + # We will select sample from whole data + eval_dataset = eval_dataset.select( + range(data_args.max_eval_samples)) + eval_datasets = [eval_dataset] if data_args.task_name == "mnli": tasks.append("mnli-mm") - eval_datasets.append(datasets["validation_mismatched"]) + mnli_mm_eval_dataset = datasets["validation_mismatched"] + if data_args.max_eval_samples is not None: + mnli_mm_eval_dataset = mnli_mm_eval_dataset.select( + range(data_args.max_eval_samples)) + eval_datasets.append(mnli_mm_eval_dataset) + for eval_dataset, _ in zip(eval_datasets, tasks): eval_result = trainer.evaluate(eval_dataset=eval_dataset) eval_results.update(eval_result) return eval_results - #case 2. when dataset is squad # Preprocessing the datasets. # Preprocessing is slighlty different for training and evaluation. diff --git a/aimet_zoo_torch/mobilebert/evaluators/mobilebert_quanteval.py b/aimet_zoo_torch/mobilebert/evaluators/mobilebert_quanteval.py index 109d930..82fc063 100644 --- a/aimet_zoo_torch/mobilebert/evaluators/mobilebert_quanteval.py +++ b/aimet_zoo_torch/mobilebert/evaluators/mobilebert_quanteval.py @@ -32,7 +32,7 @@ from aimet_zoo_torch.mobilebert.dataloader import get_datasets, eval_function -def parse_args(): +def parse_args(raw_args): """argument parser""" parser = argparse.ArgumentParser( description="Evaluating MobileBert model on GLUE datasets" @@ -54,22 +54,23 @@ def parse_args(): default=None, help="Output directory", ) - args = parser.parse_args() + args = parser.parse_args(raw_args) for arg in vars(args): print("{:30s} : {}".format(arg, getattr(args, arg))) return args +DEFAULT_CONFIG = {"MAX_EVAL_SAMPLES": None} -def main(): +def main(raw_args=None): """main function for quantization evaluation""" - args = parse_args() + args = parse_args(raw_args) logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO, ) - model = MobileBert(model_config=args.model_config) + model = MobileBert(model_config=args.model_config,args=raw_args) # get original model and tokenizer model_orig, tokenizer = model.get_model_from_pretrained() @@ -85,7 +86,7 @@ def main(): # evaluation of original model original_eval_results = eval_function( - model_orig, tokenizer, datasets, model.data_args, model.training_args + model_orig, tokenizer, datasets, model.data_args, model.training_args, max_eval_samples=DEFAULT_CONFIG["MAX_EVAL_SAMPLES"] ) # get quantsim object @@ -93,7 +94,7 @@ def main(): # evaluation of quantsim model optimized_eval_results = eval_function( - quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args + quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args, max_eval_samples=DEFAULT_CONFIG["MAX_EVAL_SAMPLES"] ) logging.info(f"***** Original Eval results *****") diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json index ae8c9e7..c3febc1 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_cola.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json index 08525c7..885178b 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mnli.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json index b9bec7e..6931e3e 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_mrpc.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json index e86236c..f20e87c 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qnli.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json index a2c58d2..1db746f 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_qqp.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json index f869f4e..d224d75 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_rte.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json index 11e100e..259b595 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_squad.json @@ -30,8 +30,8 @@ "use_auth_token":false }, "aux_args":{ - "fmodel_path":"../model/weights/pre_opt_weights", - "qmodel_path":"../model/weights/post_opt_weights" + "fmodel_path":"weights/pre_opt_weights", + "qmodel_path":"weights/post_opt_weights" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json index 774d496..cb02537 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_sst2.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json index 3ac4f31..18e3701 100644 --- a/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json +++ b/aimet_zoo_torch/mobilebert/model/model_cards/mobilebert_w8a8_stsb.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/mobilebert/model/model_definition.py b/aimet_zoo_torch/mobilebert/model/model_definition.py index d68f130..1c7cd4d 100644 --- a/aimet_zoo_torch/mobilebert/model/model_definition.py +++ b/aimet_zoo_torch/mobilebert/model/model_definition.py @@ -31,7 +31,7 @@ class MobileBert(Downloader): """model mobilebert configuration class""" #pylint:disable = import-outside-toplevel - def __init__(self, model_config=None): + def __init__(self, model_config=None, args=None): if model_config == "mobilebert_w8a8_squad": from aimet_zoo_torch.mobilebert.model.utils.utils_qa_dataclass import ( ModelArguments, @@ -44,10 +44,12 @@ def __init__(self, model_config=None): DataTrainingArguments, AuxArguments, ) - parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) - self.cfg = defaultdict(lambda: None) + self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) + self.cfg = defaultdict(lambda: None) if model_config: - config_filepath = parent_dir + "/model_cards/" + model_config + ".json" + config_filepath = os.path.join( + self.parent_dir, "model_cards", model_config + ".json" + ) with open(config_filepath) as f_in: self.cfg = json.load(f_in) Downloader.__init__( @@ -55,7 +57,7 @@ def __init__(self, model_config=None): url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], - model_dir=parent_dir, + model_dir=self.parent_dir, ) # Parse arguments parser = HfArgumentParser( @@ -66,13 +68,15 @@ def __init__(self, model_config=None): data_args, training_args, aux_args, - ) = parser.parse_args_into_dataclasses() + ) = parser.parse_args_into_dataclasses(args) self.model = None self.model_args = model_args self.data_args = data_args self.training_args = training_args self.aux_args = aux_args + self.aux_args.fmodel_path = os.path.join(self.parent_dir, self.aux_args.fmodel_path) + self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) # additional setup of the argsumetns from model_config if model_config == "mobilebert_w8a8_squad": self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] diff --git a/aimet_zoo_torch/mobilebert/model/utils/utils_nlclassifier_dataclass.py b/aimet_zoo_torch/mobilebert/model/utils/utils_nlclassifier_dataclass.py index 71ad21a..c9cbc73 100644 --- a/aimet_zoo_torch/mobilebert/model/utils/utils_nlclassifier_dataclass.py +++ b/aimet_zoo_torch/mobilebert/model/utils/utils_nlclassifier_dataclass.py @@ -124,11 +124,11 @@ class AuxArguments: Auxiliary arguments pertaining to training. """ fmodel_path: str = field( - default="../model/weights/fp.pth", + default="weights/fp.pth", metadata={"help": "Path to the full-precision model"} ) qmodel_path: str = field( - default="../model/weights/qat.ckpt", + default="weights/qat.ckpt", metadata={"help": "Path to the quantized model"} ) model_config: str = field( diff --git a/aimet_zoo_torch/mobilebert/model/utils/utils_qa_dataclass.py b/aimet_zoo_torch/mobilebert/model/utils/utils_qa_dataclass.py index f59943c..26c3646 100644 --- a/aimet_zoo_torch/mobilebert/model/utils/utils_qa_dataclass.py +++ b/aimet_zoo_torch/mobilebert/model/utils/utils_qa_dataclass.py @@ -65,11 +65,11 @@ class AuxArguments: Auxiliary arguments pertaining to training. """ fmodel_path: str = field( - default="../model/weights/fp.pth", + default="weights/fp.pth", metadata={"help": "Path to the full-precision model"} ) qmodel_path: str = field( - default="../model/weights/qat.ckpt", + default="weights/qat.ckpt", metadata={"help": "Path to the quantized model"} ) model_config: str = field( diff --git a/aimet_zoo_torch/mobilenetv2/evaluators/mobilenetv2_quanteval.py b/aimet_zoo_torch/mobilenetv2/evaluators/mobilenetv2_quanteval.py index 2d48056..f2e6edd 100644 --- a/aimet_zoo_torch/mobilenetv2/evaluators/mobilenetv2_quanteval.py +++ b/aimet_zoo_torch/mobilenetv2/evaluators/mobilenetv2_quanteval.py @@ -17,7 +17,7 @@ from aimet_zoo_torch.common.utils.utils import get_device -def arguments(): +def arguments(raw_args): """argument parser""" parser = argparse.ArgumentParser( description="Evaluation script for PyTorch ImageNet networks." @@ -50,7 +50,7 @@ def arguments(): parser.add_argument( "--use-cuda", help="Run evaluation on GPU", type=bool, default=True ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args @@ -63,11 +63,11 @@ def seed(seed_num): torch.cuda.manual_seed_all(seed_num) -def main(): +def main(raw_args=None): """main evaluation function""" # pylint:disable = too-many-locals, unused-variable seed(0) - args = arguments() + args = arguments(raw_args) device = get_device(args) eval_samples = -1 encoding_samples = 2000 diff --git a/aimet_zoo_torch/mobilevit/MobileViT.md b/aimet_zoo_torch/mobilevit/MobileViT.md index 0b519b1..0e50df4 100644 --- a/aimet_zoo_torch/mobilevit/MobileViT.md +++ b/aimet_zoo_torch/mobilevit/MobileViT.md @@ -2,11 +2,11 @@ This document describes evaluation of optimized checkpoints for mobile vision transformer (MobileViT) for image classification ## AIMET installation and setup -Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.24/packaging/install.md) (*Torch GPU* variant) before proceeding further. +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.26/packaging/install.md) (*Torch GPU* variant) before proceeding further. **NOTE** - All AIMET releases are available here: https://github.com/quic/aimet/releases -- This model has been tested using AIMET version *1.24.0* (i.e. set `release_tag="1.24.0"` in the above instructions). +- This model has been tested using AIMET version *1.24.0* (i.e. set `release_tag="1.26.0"` in the above instructions). - This model is compatible with the PyTorch GPU variant of AIMET (i.e. set `AIMET_VARIANT="torch_gpu"` in the above instructions). ### Install AIMET-Model-Zoo @@ -19,6 +19,7 @@ Clone the AIMET Model Zoo repo into your workspace: pip install accelerate==0.9.0 pip install transformers==4.21.0 pip install datasets==2.4.0 +pip install tensorboard==2.13.0 ``` @@ -51,7 +52,7 @@ Each of the {train, valid, test} directories is then expected to have 1000 subdi - To run evaluation with QuantSim in AIMET, use the following ```bash -python transformer_imageclassification.py \ +python mobilevit_quanteval.py \ --model_config \ --per_device_eval_batch_size \ --train_dir \ diff --git a/aimet_zoo_torch/mobilevit/dataloader/dataloaders.py b/aimet_zoo_torch/mobilevit/dataloader/dataloaders.py index 023a921..04f86d0 100644 --- a/aimet_zoo_torch/mobilevit/dataloader/dataloaders.py +++ b/aimet_zoo_torch/mobilevit/dataloader/dataloaders.py @@ -9,7 +9,8 @@ # ============================================================================= #pylint: skip-file """ module for getting dataloders""" - +import os +import pathlib from PIL import Image from datasets import load_dataset import torch @@ -24,7 +25,24 @@ ToTensor, ) -def get_dataloaders(config,feature_extractor,interpolate=False): +# pylint: disable-msg=R0902 +class DataConfig: + """adding hardcoded values into args from parseargs() and return args""" + + def __init__(self, args): + self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent.parent) + # import pdb + # pdb.set_trace() + self.dataset_name = os.path.join(self.parent_dir,"dataloader/utils/imagenet.py") + self.max_eval_samples = None + self.max_train_samples = None + self.clamp_quantizer = False + self.per_device_train_batch_size = 8 + self.image_normalization = False + for arg in vars(args): + setattr(self, arg, getattr(args, arg)) + +def get_dataloaders(args,feature_extractor,interpolate=False): """Get train_dataloader and val_dataloader @@ -33,13 +51,16 @@ def get_dataloaders(config,feature_extractor,interpolate=False): #Load aimet model model, feature_extractor, interpolate = load_pretrained_model( - config, labels, label2id, id2label + args, labels, label2id, id2label ) """ - # get dataset from config + # hardcoded values for args + args = DataConfig(args) + + # get dataset from args - dataset = get_dataset(config) + dataset = get_dataset(args) # Prepare label mappings. # We'll include these in the model's config to get human readable labels @@ -63,7 +84,7 @@ def get_dataloaders(config,feature_extractor,interpolate=False): CenterCrop(feature_extractor.size), ToTensor(), ] - if config.image_normalization: + if args.image_normalization: _train_transforms.append(normalize) _val_transforms.append(normalize) train_transforms = Compose(_train_transforms) @@ -95,19 +116,19 @@ def preprocess_val(example_batch): image.convert("RGB")) for image in example_batch["image"]] return example_batch - if config.max_train_samples is not None: + if args.max_train_samples is not None: dataset["train"] = ( dataset["train"] - .shuffle(seed=config.seed) - .select(range(config.max_train_samples)) + .shuffle(seed=args.seed) + .select(range(args.max_train_samples)) ) # Set the training transforms train_dataset = dataset["train"].with_transform(preprocess_train) - if config.max_eval_samples is not None: + if args.max_eval_samples is not None: dataset["validation"] = ( dataset["validation"] - .shuffle(seed=config.seed) - .select(range(config.max_eval_samples)) + .shuffle(seed=args.seed) + .select(range(args.max_eval_samples)) ) # Set the validation transforms eval_dataset = dataset["validation"].with_transform(preprocess_val) @@ -130,12 +151,12 @@ def collate_fn(examples): train_dataset, shuffle=True, collate_fn=collate_fn, - batch_size=config.per_device_train_batch_size, + batch_size=args.per_device_train_batch_size, ) eval_dataloader = DataLoader( eval_dataset, collate_fn=collate_fn, - batch_size=config.per_device_eval_batch_size, + batch_size=args.per_device_eval_batch_size, ) def eval_function(model,args): @@ -169,10 +190,10 @@ def eval_function(model,args): return train_dataloader,eval_dataloader,eval_function -def get_dataset(config): +def get_dataset(args): """get imagenet dataset Parameters: - config: location of imagenet train and validation dataset + args: location of imagenet train and validation dataset Returns: dataset: imagenet dataset """ @@ -184,10 +205,10 @@ def get_dataset(config): # download the dataset. # imagenet custom script loader data_files = {} - data_files["train"] = config.train_dir + data_files["train"] = args.train_dir # if config.validation_dir is not None: - data_files["validation"] = config.validation_dir + data_files["validation"] = args.validation_dir # if config.dataset_name.endswith(".py"): - dataset = load_dataset(config.dataset_name, data_dir=data_files) + dataset = load_dataset(args.dataset_name, data_dir=data_files) return dataset diff --git a/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py b/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py index 132293a..e33536f 100644 --- a/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py +++ b/aimet_zoo_torch/mobilevit/evaluators/mobilevit_quanteval.py @@ -30,15 +30,15 @@ ) -def parse_args(): +def parse_args(raw_args): """argument parser""" parser = argparse.ArgumentParser( description="Evaluating VIT/MobileVIT Transformers model on an imagenet dataset" ) parser.add_argument( "--model_config", - default="vit_w8a8", - help="choice [vit_w8a8]", + default="mobilevit_w8a8", + help="choice [mobilevit_w8a8]", ) parser.add_argument( "--train_dir", @@ -58,31 +58,20 @@ def parse_args(): default=8, help="Batch size (per device) for the evaluation dataloader.", ) - args = parser.parse_args() + parser.add_argument( + "--seed", + type=int, + default=2022, + help="training seed", + ) + args = parser.parse_args(raw_args) return args -# pylint: disable-msg=R0902 -class DataConfig: - """adding hardcoded values into args from parseargs() and return config object""" - - def __init__(self, args): - self.dataset_name = "../dataloader/utils/imagenet.py" - self.seed = 2022 - self.max_eval_samples = None - self.max_train_samples = None - self.clamp_quantizer = False - self.per_device_train_batch_size = 8 - self.image_normalization = False - for arg in vars(args): - setattr(self, arg, getattr(args, arg)) - - -def main(): +def main(raw_args=None): """Evaluation main function""" - args = parse_args() - config = DataConfig(args) + args = parse_args(raw_args) # Initialize the accelerator. We will let the accelerator # handle device placement for us in this example. # If we're using tracking, we also need to initialize it here @@ -105,8 +94,8 @@ def main(): transformers.utils.logging.set_verbosity_error() # If passed along, set the training seed now. - if config.seed is not None: - set_seed(config.seed) + if args.seed is not None: + set_seed(args.seed) accelerator.wait_for_everyone() @@ -118,7 +107,7 @@ def main(): # load modularized eval_function and dataloaders train_dataloader, eval_dataloader, eval_function = get_dataloaders( - config, feature_extractor + args, feature_extractor ) # Prepare everything with our `accelerator`. @@ -177,7 +166,13 @@ def main(): logger.info( f"Optimized Model | 8-bit Environment | perplexity: {quantized_model_performance_int8:.4f}" ) - + + return { + 'original_model_performance_fp32':original_model_performance_fp32, + 'original_model_performance_int8':original_model_performance_int8, + 'quantized_model_performance_fp32':quantized_model_performance_fp32, + 'quantized_model_performance_int8':quantized_model_performance_int8 + } if __name__ == "__main__": main() diff --git a/aimet_zoo_torch/mobilevit/model/model_cards/mobilevit_w8a8.json b/aimet_zoo_torch/mobilevit/model/model_cards/mobilevit_w8a8.json index 66f00a6..8f9b951 100644 --- a/aimet_zoo_torch/mobilevit/model/model_cards/mobilevit_w8a8.json +++ b/aimet_zoo_torch/mobilevit/model/model_cards/mobilevit_w8a8.json @@ -3,12 +3,12 @@ "framework": "pytorch", "task": "image classification", "model_args": { - "quantized":{"model_name_or_path": "../model/weights/downloaded_weights"}, - "original":{"model_name_or_path": "apple/mobilevit-small"}, - "dataset_name": "../model/utils/imagenet.py", + "quantized":{"model_name_or_path": "weights/downloaded_weights"}, + "original":{"model_name_or_path": "apple/mobilevit-small"}, + "dataset_name": "utils/imagenet.py", "higher_resolution": "False", "ignore_mismatched_sizes": "False", - "config_file": "../model/weights/aimet_config", + "config_file": "weights/aimet_config", "clamp_quantizer": "False", "clamping_value": "30.0" }, diff --git a/aimet_zoo_torch/mobilevit/model/model_definition.py b/aimet_zoo_torch/mobilevit/model/model_definition.py index f0d6152..b8c4dc5 100644 --- a/aimet_zoo_torch/mobilevit/model/model_definition.py +++ b/aimet_zoo_torch/mobilevit/model/model_definition.py @@ -13,6 +13,7 @@ import os import csv from collections import defaultdict +import pathlib import torch from transformers import AutoConfig as Config from transformers import AutoFeatureExtractor as FeatureExtractor @@ -35,20 +36,32 @@ def __init__(self, model_config=None, quantized=False): model_config quantized """ - parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) + self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = defaultdict(lambda: None) if model_config: - config_filepath = parent_dir + "/model_cards/" + model_config + ".json" + config_filepath = os.path.join( + self.parent_dir, "model_cards", model_config + ".json" + ) with open(config_filepath) as f_in: self.cfg = json.load(f_in) Downloader.__init__( self, tar_url_post_opt_weights=self.cfg["artifacts"]["tar_url_post_opt_weights"], url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], - model_dir=parent_dir, + model_dir=self.parent_dir, ) self.model = None self.quantized = quantized + if self.quantized: + self.model_name_or_path = os.path.join( + self.parent_dir, self.cfg["model_args"]["quantized"]["model_name_or_path"] + ) + else: + self.model_name_or_path = self.cfg["model_args"]["original"]["model_name_or_path"] + + self.config_file = os.path.join( + self.parent_dir, self.cfg["model_args"]["config_file"] + ) def get_model_from_pretrained(self): """get original or optmized model @@ -61,33 +74,17 @@ def get_model_from_pretrained(self): self._download_tar_post_opt_weights() self._download_aimet_config() - if self.quantized: - model_name_or_path = self.cfg["model_args"]["quantized"][ - "model_name_or_path" - ] - else: - model_name_or_path = self.cfg["model_args"]["original"][ - "model_name_or_path" - ] - - config = Config.from_pretrained(model_name_or_path) + config = Config.from_pretrained(self.model_name_or_path) config.return_dict = False - self.model = MobileVitModel.from_pretrained(model_name_or_path, config=config) + self.model = MobileVitModel.from_pretrained(self.model_name_or_path, config=config) return self.model def get_feature_extractor_from_pretrained(self): """get feature extractor from pretrained model""" - if self.quantized: - model_name_or_path = self.cfg["model_args"]["quantized"][ - "model_name_or_path" - ] - else: - model_name_or_path = self.cfg["model_args"]["original"][ - "model_name_or_path" - ] + feature_extractor = FeatureExtractor.from_pretrained( - model_name_or_path, + self.model_name_or_path, ) return feature_extractor @@ -100,15 +97,6 @@ def get_quantsim(self, train_dataloader, eval_dataloader, eval_function): Returns: quant_sim: """ - if self.quantized: - model_name_or_path = self.cfg["model_args"]["quantized"][ - "model_name_or_path" - ] - else: - model_name_or_path = self.cfg["model_args"]["original"][ - "model_name_or_path" - ] - metric = datasets.load_metric("accuracy") dummy_input = self._get_dummy_input(train_dataloader) @@ -147,13 +135,13 @@ def get_quantsim(self, train_dataloader, eval_dataloader, eval_function): "quantization_configuration" ]["param_bw"], in_place=True, - config_file=self.cfg["model_args"]["config_file"], + config_file=self.config_file, ) quant_sim.compute_encodings(eval_function, [10, eval_dataloader, metric]) # load encodings if there is encodings.csv - self._load_encoding_data(quant_sim, model_name_or_path) + self._load_encoding_data(quant_sim, self.model_name_or_path) return quant_sim @staticmethod diff --git a/aimet_zoo_torch/regnet/evaluator/regnet_quanteval.py b/aimet_zoo_torch/regnet/evaluator/regnet_quanteval.py index d45631d..1f98295 100644 --- a/aimet_zoo_torch/regnet/evaluator/regnet_quanteval.py +++ b/aimet_zoo_torch/regnet/evaluator/regnet_quanteval.py @@ -18,7 +18,7 @@ from aimet_zoo_torch.regnet import RegNet -def arguments(): +def arguments(raw_args): """Parse input arguments""" parser = argparse.ArgumentParser( description="script for classification model quantization" @@ -35,13 +35,13 @@ def arguments(): "--dataset-path", help="path to evaluation dataset", type=str, required=True ) parser.add_argument("--use-cuda", help="Use cuda", default=True, type=bool) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args -def main(): +def main(raw_args=None): """Run evaluations""" - args = arguments() + args = arguments(raw_args) # Dataloaders encoding_dataloader = ImageNetDataLoader( diff --git a/aimet_zoo_torch/resnet/ResNet.md b/aimet_zoo_torch/resnet/ResNet.md index 4deaf11..7ff81f7 100644 --- a/aimet_zoo_torch/resnet/ResNet.md +++ b/aimet_zoo_torch/resnet/ResNet.md @@ -56,10 +56,12 @@ python resnet_quanteval.py\ Available model configurations are: - resnet18_w8a8 - resnet50_w8a8 +- resnet50_w8a16 - resnet101_w8a8 --- ## Quantization Configuration + W8A8 optimization The following configuration has been used for the above models for W8A8 quantization: @@ -72,13 +74,16 @@ The following configuration has been used for the above models for W8A8 quantiza - Cross layer equalization and Adaround in per channel mode has been applied for ResNet18, ResNet50 to get the best W8A8 optimized checkpoint - Cross layer equalization and Adaround in per tensor mode has been applied for ResNet101 to get the best W8A8 optimized checkpoint -W4A8 optimization -The following configuration has been used for the above models for INT4 quantization: -- Weight quantization: 4 bits, symmetric quantization +W8A16 optimization + +The following configuration has been used for the above models for W8A16 quantization: +- Weight quantization: 8 bits, symmetric quantization - Bias parameters are not quantized -- Activation quantization: 8 bits, asymmetric quantization +- Activation quantization: 16 bits, asymmetric quantization - Model inputs are quantized - 2000 images from the calibration dataset were used for computing encodings - TF_enhanced was used as quantization scheme -- Cross layer equalization and Adaround in per channel mode has been applied for all the models to get the best INT4 optimized checkpoint +- Batch Norm Folding in per channel mode has been applied for ResNet50 to get the best W8A16 optimized checkpoint + + diff --git a/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py b/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py index 57a08f4..cde8a49 100644 --- a/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py +++ b/aimet_zoo_torch/resnet/evaluator/resnet_quanteval.py @@ -19,7 +19,7 @@ def arguments(raw_args): """ Parse input arguments """ parser = argparse.ArgumentParser(description='script for classification model quantization') parser.add_argument('--model-config', help='model configuration to use', default="resnet50_w8a8", - choices = ['resnet18_w4a8', 'resnet18_w8a8', 'resnet50_w4a8', 'resnet50_w8a8', 'resnet101_w8a8'], + choices = ['resnet18_w4a8', 'resnet18_w8a8', 'resnet50_w4a8', 'resnet50_w8a8', 'resnet50_w8a16', 'resnet101_w8a8'], type=str, required=True) parser.add_argument('--dataset-path', help='path to evaluation dataset',type=str, required=True) parser.add_argument('--use-cuda', help='Use cuda', default=True, type=bool) @@ -49,7 +49,7 @@ def main(raw_args=None): quant_acc = eval_func(model = sim.model.cuda(), dataloader = eval_dataloader) print(f'Quantized quantized accuracy: {quant_acc:0.3f}%') - return quant_acc + return {'fp32_acc':fp32_acc, 'quant_acc':quant_acc} if __name__ == '__main__': main() diff --git a/aimet_zoo_torch/resnet/model/model_cards/resnet50_w8a16.json b/aimet_zoo_torch/resnet/model/model_cards/resnet50_w8a16.json new file mode 100644 index 0000000..1e841f7 --- /dev/null +++ b/aimet_zoo_torch/resnet/model/model_cards/resnet50_w8a16.json @@ -0,0 +1,27 @@ +{ + "name": "ResNet50", + "framework": "pytorch", + "task": "image classification", + "model_args": { + "num_classes": 1000 + }, + "input_shape": [null, 3, 224, 224], + "training_dataset": "ImageNet", + "optimization_config": { + "quantization_configuration": + { + "param_bw": 8, + "output_bw": 16, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "techniques": ["bnfold"] + } + }, + "artifacts": { + "url_pre_opt_weights": null, + "url_post_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_resnet50_w8a16/resnet50_w8a16_state_dict.pth", + "url_adaround_encodings": null, + "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/torch_resnet50_w8a16/resnet50_w8a16_torch.encodings", + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/50cfafe353b530d81c52188151c418ba16e92261/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" + } +} diff --git a/aimet_zoo_torch/resnext/evaluator/resnext_quanteval.py b/aimet_zoo_torch/resnext/evaluator/resnext_quanteval.py index c424b17..699dd1e 100644 --- a/aimet_zoo_torch/resnext/evaluator/resnext_quanteval.py +++ b/aimet_zoo_torch/resnext/evaluator/resnext_quanteval.py @@ -18,7 +18,7 @@ from aimet_zoo_torch.common.utils.utils import get_device -def arguments(): +def arguments(raw_args): """Parse input arguments""" parser = argparse.ArgumentParser( description="script for classification model quantization" @@ -35,13 +35,13 @@ def arguments(): "--dataset-path", help="path to evaluation dataset", type=str, required=True ) parser.add_argument("--use-cuda", help="Use cuda", default=True, type=bool) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args -def main(): +def main(raw_args=None): """Run evaluations""" - args = arguments() + args = arguments(raw_args) device = get_device(args) # Dataloaders eval_dataloader = ImageNetDataLoader(args.dataset_path, image_size=224).data_loader diff --git a/aimet_zoo_torch/resnext/model/model_cards/resnext101_w8a8.json b/aimet_zoo_torch/resnext/model/model_cards/resnext101_w8a8.json index 3c917bd..a0914a9 100644 --- a/aimet_zoo_torch/resnext/model/model_cards/resnext101_w8a8.json +++ b/aimet_zoo_torch/resnext/model/model_cards/resnext101_w8a8.json @@ -21,7 +21,7 @@ "url_pre_opt_weights": null, "url_post_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_resnext101/resnext101_w8a8_state_dict.pth", "url_adaround_encodings": null, - "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/torch_resnext101/resnext101_w8a8_state_dict.encodings", + "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/torch_resnext101/resnext101_w8a8.encodings", "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.24/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config.json" } } diff --git a/aimet_zoo_torch/roberta/dataloader/dataloaders.py b/aimet_zoo_torch/roberta/dataloader/dataloaders.py index aeeb0e2..dccca7e 100644 --- a/aimet_zoo_torch/roberta/dataloader/dataloaders.py +++ b/aimet_zoo_torch/roberta/dataloader/dataloaders.py @@ -303,7 +303,9 @@ def preprocess_function(examples): return datasets -def eval_function(model,tokenizer,datasets,data_args,training_args): +def eval_function(model,tokenizer,datasets,data_args,training_args,max_eval_samples=None): + if max_eval_samples is not None: + data_args.max_eval_samples = max_eval_samples ## case 1. when dataset is glue if hasattr(data_args,'task_name'): train_dataset = datasets["train"] @@ -366,10 +368,18 @@ def compute_metrics(p: EvalPrediction): # Loop to handle MNLI double evaluation (matched, mis-matched) tasks = [data_args.task_name] - eval_datasets = [eval_dataset] + if data_args.max_eval_samples is not None: + # We will select sample from whole data + eval_dataset = eval_dataset.select( + range(data_args.max_eval_samples)) + eval_datasets = [eval_dataset] if data_args.task_name == "mnli": tasks.append("mnli-mm") - eval_datasets.append(datasets["validation_mismatched"]) + mnli_mm_eval_dataset = datasets["validation_mismatched"] + if data_args.max_eval_samples is not None: + mnli_mm_eval_dataset = mnli_mm_eval_dataset.select( + range(data_args.max_eval_samples)) + eval_datasets.append(mnli_mm_eval_dataset) for eval_dataset, _ in zip(eval_datasets, tasks): eval_result = trainer.evaluate(eval_dataset=eval_dataset) eval_results.update(eval_result) diff --git a/aimet_zoo_torch/roberta/evaluators/roberta_quanteval.py b/aimet_zoo_torch/roberta/evaluators/roberta_quanteval.py index faf7585..e5c6207 100644 --- a/aimet_zoo_torch/roberta/evaluators/roberta_quanteval.py +++ b/aimet_zoo_torch/roberta/evaluators/roberta_quanteval.py @@ -32,7 +32,7 @@ from aimet_zoo_torch.roberta.dataloader import get_datasets, eval_function -def parse_args(): +def parse_args(raw_args): """argument parser""" parser = argparse.ArgumentParser( description="Evaluating Bert model on GLUE datasets" @@ -54,22 +54,23 @@ def parse_args(): default=None, help="Output directory", ) - args = parser.parse_args() + args = parser.parse_args(raw_args) for arg in vars(args): print("{:30s} : {}".format(arg, getattr(args, arg))) return args +DEFAULT_CONFIG = {"MAX_EVAL_SAMPLES": None} -def main(): +def main(raw_args=None): """main function for quantization evaluation""" - args = parse_args() + args = parse_args(raw_args) logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO, ) - model = Roberta(model_config=args.model_config) + model = Roberta(model_config=args.model_config,args=raw_args) # get original model and tokenizer model_orig, tokenizer = model.get_model_from_pretrained() @@ -85,7 +86,7 @@ def main(): # evaluation of original model original_eval_results = eval_function( - model_orig, tokenizer, datasets, model.data_args, model.training_args + model_orig, tokenizer, datasets, model.data_args, model.training_args,max_eval_samples=DEFAULT_CONFIG['MAX_EVAL_SAMPLES'] ) # get quantsim object @@ -93,7 +94,7 @@ def main(): # evaluation of quantsim model optimized_eval_results = eval_function( - quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args + quantsim_model.model, tokenizer, datasets, model.data_args, model.training_args,max_eval_samples=DEFAULT_CONFIG['MAX_EVAL_SAMPLES'] ) logging.info(f"***** Original Eval results *****") diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_cola.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_cola.json index c6ca00d..ed3d50d 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_cola.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_cola.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mnli.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mnli.json index 8239d90..3190cd1 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mnli.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mnli.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mrpc.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mrpc.json index 8fc09a2..434a226 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mrpc.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_mrpc.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qnli.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qnli.json index 07cebac..ede20ca 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qnli.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qnli.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qqp.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qqp.json index 961d966..b0cd7a0 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qqp.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_qqp.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_rte.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_rte.json index bb2a3df..4360015 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_rte.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_rte.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_sst2.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_sst2.json index 724169e..b45c96b 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_sst2.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_sst2.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_stsb.json b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_stsb.json index 00a7b6e..66f6008 100644 --- a/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_stsb.json +++ b/aimet_zoo_torch/roberta/model/model_cards/roberta_w8a8_stsb.json @@ -23,8 +23,8 @@ "attention_probs_dropout_prob":0.1 }, "aux_args":{ - "fmodel_path":"../model/weights/fp.pth", - "qmodel_path":"../model/weights/qat.ckpt" + "fmodel_path":"weights/fp.pth", + "qmodel_path":"weights/qat.ckpt" }, "optimization_config": { "quantization_configuration": diff --git a/aimet_zoo_torch/roberta/model/model_definition.py b/aimet_zoo_torch/roberta/model/model_definition.py index 2a0c8fd..da18569 100644 --- a/aimet_zoo_torch/roberta/model/model_definition.py +++ b/aimet_zoo_torch/roberta/model/model_definition.py @@ -34,7 +34,7 @@ class Roberta(Downloader): """model roberta configuration class""" #pylint:disable = import-outside-toplevel - def __init__(self, model_config=None): + def __init__(self, model_config=None, args=None): if model_config == "roberta_w8a8_squad": from aimet_zoo_torch.roberta.model.utils.utils_qa_dataclass import ( ModelArguments, @@ -47,10 +47,12 @@ def __init__(self, model_config=None): DataTrainingArguments, AuxArguments, ) - parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) + self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = defaultdict(lambda: None) if model_config: - config_filepath = parent_dir + "/model_cards/" + model_config + ".json" + config_filepath = os.path.join( + self.parent_dir, "model_cards", model_config + ".json" + ) with open(config_filepath) as f_in: self.cfg = json.load(f_in) Downloader.__init__( @@ -58,7 +60,7 @@ def __init__(self, model_config=None): url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], - model_dir=parent_dir, + model_dir=self.parent_dir, ) # Parse arguments parser = HfArgumentParser( @@ -69,13 +71,15 @@ def __init__(self, model_config=None): data_args, training_args, aux_args, - ) = parser.parse_args_into_dataclasses() + ) = parser.parse_args_into_dataclasses(args) self.model = None self.model_args = model_args self.data_args = data_args self.training_args = training_args self.aux_args = aux_args + self.aux_args.fmodel_path = os.path.join(self.parent_dir, self.aux_args.fmodel_path) + self.aux_args.qmodel_path = os.path.join(self.parent_dir, self.aux_args.qmodel_path) # additional setup of the argsumetns from model_config if model_config == "roberta_w8a8_squad": self.data_args.dataset_name = self.cfg["data_training_args"]["dataset_name"] diff --git a/aimet_zoo_torch/roberta/model/utils/utils_nlclassifier_dataclass.py b/aimet_zoo_torch/roberta/model/utils/utils_nlclassifier_dataclass.py index f30bf93..da1676f 100644 --- a/aimet_zoo_torch/roberta/model/utils/utils_nlclassifier_dataclass.py +++ b/aimet_zoo_torch/roberta/model/utils/utils_nlclassifier_dataclass.py @@ -125,11 +125,11 @@ class AuxArguments: Auxiliary arguments pertaining to training. """ fmodel_path: str = field( - default="../model/weights/fp.pth", + default="weights/fp.pth", metadata={"help": "Path to the full-precision model"} ) qmodel_path: str = field( - default="../model/weights/qat.ckpt", + default="weights/qat.ckpt", metadata={"help": "Path to the quantized model"} ) model_config: str = field( diff --git a/aimet_zoo_torch/roberta/model/utils/utils_qa_dataclass.py b/aimet_zoo_torch/roberta/model/utils/utils_qa_dataclass.py index 2a53959..d62d0bd 100644 --- a/aimet_zoo_torch/roberta/model/utils/utils_qa_dataclass.py +++ b/aimet_zoo_torch/roberta/model/utils/utils_qa_dataclass.py @@ -66,11 +66,11 @@ class AuxArguments: Auxiliary arguments pertaining to training. """ fmodel_path: str = field( - default="../model/weights/fp.pth", + default="weights/fp.pth", metadata={"help": "Path to the full-precision model"} ) qmodel_path: str = field( - default="../model/weights/qat.ckpt", + default="weights/qat.ckpt", metadata={"help": "Path to the quantized model"} ) model_config: str = field( diff --git a/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py b/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py index 36fbb91..28afaff 100644 --- a/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py +++ b/aimet_zoo_torch/ssd_mobilenetv2/evaluators/ssd_mobilenetv2_quanteval.py @@ -71,7 +71,7 @@ def download_labels(): ) -def arguments(): +def arguments(raw_args): # pylint: disable = redefined-outer-name """parses command line arguments""" parser = argparse.ArgumentParser(description="SSD Evaluation on VOC Dataset.") @@ -90,7 +90,7 @@ def arguments(): parser.add_argument("--default-output-bw", type=int, default=8) parser.add_argument("--default-param-bw", type=int, default=8) parser.add_argument("--use-cuda", type=bool, default=True) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args @@ -121,11 +121,11 @@ def work_init(work_id): np.random.seed(seed + work_id) -def model_eval(args, predictor, dataset): +def model_eval(args, predictor, dataset, num_samples): """ model evalution function""" # pylint: disable = redefined-outer-name, unused-variable, unused-argument aimet_dataset = copy.deepcopy(dataset) - aimet_dataset.ids = aimet_dataset.ids[:500] + aimet_dataset.ids = aimet_dataset.ids[:num_samples] calib_dataset = CalibrationDataset(aimet_dataset) data_loader_kwargs = {"worker_init_fn": work_init, "num_workers": 0} batch_size = 1 @@ -183,6 +183,7 @@ def group_annotation_by_class(dataset): all_gt_boxes[class_index][image_id] = torch.stack( all_gt_boxes[class_index][image_id] ) + #pylint:disable = consider-using-dict-items for class_index in all_difficult_cases: for image_id in all_difficult_cases[class_index]: all_gt_boxes[class_index][image_id] = torch.tensor( @@ -247,12 +248,13 @@ def compute_average_precision_per_class( return measurements.compute_average_precision(precision, recall) -def evaluate_predictor(predictor): +def evaluate_predictor(predictor,dataset,class_names,eval_path,annotation_stats,config,num_samples): """ :param predictor: :return: Average precision per classes for the given predictor """ # pylint: disable = too-many-locals, redefined-outer-name + true_case_stat, all_gb_boxes, all_difficult_cases = annotation_stats results = [] for i in tqdm(range(len(dataset))): image = dataset.get_image(i) @@ -269,6 +271,9 @@ def evaluate_predictor(predictor): dim=1, ) ) + if num_samples is not None and i > num_samples: + break + results = torch.cat(results) for class_index, class_name in enumerate(class_names): if class_index == 0: @@ -315,20 +320,25 @@ def __init__(self, args): for arg in vars(args): setattr(self, arg, getattr(args, arg)) +DEFAULT_CONFIG = {"num_samples_cal": 500, "num_samples_eval": None} -if __name__ == "__main__": - args = arguments() +#pylint:disable = too-many-local-variables +def main(raw_args=None): + """main evaluation function""" + args = arguments(raw_args) config = ModelConfig(args) - download_labels() + #download_labels() eval_path = pathlib.Path("./eval_results") eval_path.mkdir(exist_ok=True) - #pylint:disable = consider-using-with class_names = [name.strip() for name in open("voc-model-labels.txt").readlines()] device = get_device(args) - #pylint:disable = no-member + #pylint: disable = no-member dataset = VOCDataset(config.dataset_path, is_test=True) - true_case_stat, all_gb_boxes, all_difficult_cases = group_annotation_by_class( + #true_case_stat, all_gb_boxes, all_difficult_cases = group_annotation_by_class( + # dataset + #) + annotation_stats = group_annotation_by_class( dataset ) @@ -339,7 +349,7 @@ def __init__(self, args): model_fp32.model, nms_method="hard", device=device ) sim_fp32 = model_fp32.get_quantsim(quantized=False) - eval_func_fp32 = model_eval(config, predictor_orig_fp32, dataset) + eval_func_fp32 = model_eval(config, predictor_orig_fp32, dataset, DEFAULT_CONFIG['num_samples_cal']) predictor_sim_fp32 = create_mobilenetv2_ssd_lite_predictor( sim_fp32.model, nms_method=config.nms_method, device=device ) @@ -348,11 +358,12 @@ def __init__(self, args): print("Initializing Optimized Model") model_int8 = SSDMobileNetV2(model_config=args.model_config) model_int8.from_pretrained(quantized=True) + predictor_orig_int8 = create_mobilenetv2_ssd_lite_predictor( model_int8.model, nms_method=config.nms_method, device=device ) sim_int8 = model_int8.get_quantsim(quantized=True) - eval_func_int8 = model_eval(config, predictor_orig_int8, dataset) + eval_func_int8 = model_eval(config, predictor_orig_int8, dataset, DEFAULT_CONFIG['num_samples_cal']) predictor_sim_int8 = create_mobilenetv2_ssd_lite_predictor( sim_int8.model, nms_method=config.nms_method, device=device ) @@ -360,22 +371,22 @@ def __init__(self, args): # Original FP32 model on FP32 device print("Computing Original Model on FP32 device") - aps = evaluate_predictor(predictor_orig_fp32) + aps = evaluate_predictor(predictor_orig_fp32,dataset,class_names,eval_path,annotation_stats,config,DEFAULT_CONFIG['num_samples_eval']) mAP_fp32model_fp32env = sum(aps) / len(aps) # Original FP32 model on INT8 device print("Computing Original Model on INT8 device") - aps = evaluate_predictor(predictor_sim_fp32) + aps = evaluate_predictor(predictor_sim_fp32,dataset,class_names,eval_path,annotation_stats,config,DEFAULT_CONFIG['num_samples_eval']) mAP_fp32model_int8env = sum(aps) / len(aps) # Quantized INT8 model on FP32 device print("Computing Optimized Model on FP32 device") - aps = evaluate_predictor(predictor_orig_int8) + aps = evaluate_predictor(predictor_orig_int8,dataset,class_names,eval_path,annotation_stats,config,DEFAULT_CONFIG['num_samples_eval']) mAP_int8model_fp32env = sum(aps) / len(aps) # Quantized INT8 model on INT8 device print("Computing Optimized Model on INT8 device") - aps = evaluate_predictor(predictor_sim_int8) + aps = evaluate_predictor(predictor_sim_int8,dataset,class_names,eval_path,annotation_stats,config,DEFAULT_CONFIG['num_samples_eval']) mAP_int8model_int8env = sum(aps) / len(aps) print("\n\n") @@ -384,3 +395,6 @@ def __init__(self, args): print(f"Original Model on INT8 device | mAP: {mAP_fp32model_int8env:.4f}") print(f"Optimized Model on FP32 device | mAP: {mAP_int8model_fp32env:.4f}") print(f"Optimized Model on INT8 device | mAP: {mAP_int8model_int8env:.4f}") + +if __name__ == "__main__": + main() diff --git a/aimet_zoo_torch/ssd_res50/evaluators/ssd_res50_quanteval.py b/aimet_zoo_torch/ssd_res50/evaluators/ssd_res50_quanteval.py index 507e38b..b55ed15 100644 --- a/aimet_zoo_torch/ssd_res50/evaluators/ssd_res50_quanteval.py +++ b/aimet_zoo_torch/ssd_res50/evaluators/ssd_res50_quanteval.py @@ -53,7 +53,7 @@ from src.transform import SSDTransformer # pylint:disable = import-error from pycocotools.cocoeval import COCOeval # pylint:disable = import-error -def get_args(): +def get_args(raw_args): """argument parser""" #pylint:disable = redefined-outer-name parser = argparse.ArgumentParser("Evaluation script for quantized SSD Res50 Model") @@ -81,15 +81,16 @@ def get_args(): "--use-cuda", help="Use GPU for evaluation", action="store_true" ) - args = parser.parse_args() + args = parser.parse_args(raw_args) return args -def ssd_res50_quanteval(args): +def main(raw_args=None): """ Evaluation function for SSD Res50 quantized model """ #pylint:disable = redefined-outer-name + args=get_args(raw_args) if args.use_cuda: if torch.cuda.is_available(): device = torch.device("cuda") @@ -216,5 +217,4 @@ def evaluate(model, test_loader, encoder, args, device): if __name__ == "__main__": - args = get_args() - ssd_res50_quanteval(args) + main() diff --git a/aimet_zoo_torch/ssd_res50/model/model_cards/ssd_res50_w8a8.json b/aimet_zoo_torch/ssd_res50/model/model_cards/ssd_res50_w8a8.json index 30741fd..1f1876b 100644 --- a/aimet_zoo_torch/ssd_res50/model/model_cards/ssd_res50_w8a8.json +++ b/aimet_zoo_torch/ssd_res50/model/model_cards/ssd_res50_w8a8.json @@ -23,4 +23,4 @@ "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/torch_ssd_res50/SSD_Res50_torch.encodings", "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.23/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" } -} \ No newline at end of file +} diff --git a/aimet_zoo_torch/uniformer_classification/UniFormer.md b/aimet_zoo_torch/uniformer_classification/UniFormer.md new file mode 100644 index 0000000..1ac73e3 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/UniFormer.md @@ -0,0 +1,51 @@ +# Uniformer for Image Classification + +## Environment Setup + +### Setup AI Model Efficiency Toolkit (AIMET) +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.26/packaging/install.md) before proceeding further. +This model was tested with the `torch_gpu` variant of AIMET 1.26. + +### Environment Setup +Append the repo location to your `PYTHONPATH` with the following: + ```bash + export PYTHONPATH=$PYTHONPATH: + ``` + +### Dataset +The ImageNet 2012 Challenge (ILSVRC2012) dataset can be obtained from : + - https://www.image-net.org/download.php + + +--- + +## Usage +```bash +python3 aimet_zoo_torch/uniformer_classification/evaluators/uniformer_classification_quanteval.py \ + --model-config \ + --dataset-path \ + --batch-size \ +``` + +Available model configurations are: +- uniformer_classification_w8a8 + +--- + + +## Model checkpoint and configuration + +Individual artifacts can be obtained from: + - https://github.com/quic/aimet-model-zoo/releases/tag/torch_uniformer_classification + +--- + +## Quantization Configuration +The following configuration has been used for both W4A8 and W8A8 variants: +- Weight quantization: 8 bits, per tensor symmetric quantization +- Bias parameters are not quantized +- Activation quantization: 8 bits, asymmetric quantization +- Model inputs are quantized +- training_range_learning_with_tf_init was used as quantization scheme +- Batch Norm Fold has been applied on optimized checkpoint +- Quantization Aware Training has been performed on the optimized checkpoint diff --git a/aimet_zoo_torch/uniformer_classification/__init__.py b/aimet_zoo_torch/uniformer_classification/__init__.py new file mode 100644 index 0000000..8db3a27 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/__init__.py @@ -0,0 +1,2 @@ +""" loading downloader class """ +from .model.model_definition import UniformerClassification diff --git a/aimet_zoo_torch/uniformer_classification/dataloader/__init__.py b/aimet_zoo_torch/uniformer_classification/dataloader/__init__.py new file mode 100644 index 0000000..7661cb7 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/dataloader/__init__.py @@ -0,0 +1,2 @@ +"""loading dataloader and evaluation function""" +from .dataloaders_and_eval_func import get_dataloaders_and_eval_func diff --git a/aimet_zoo_torch/uniformer_classification/dataloader/dataloaders_and_eval_func.py b/aimet_zoo_torch/uniformer_classification/dataloader/dataloaders_and_eval_func.py new file mode 100644 index 0000000..f4d2e56 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/dataloader/dataloaders_and_eval_func.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +""" module for getting dataloader and evaluation function for deeplabv3 """ + +import torch +from aimet_zoo_torch.uniformer_classification.model.image_classification.engine import evaluate +from aimet_zoo_torch.uniformer_classification.model.image_classification.datasets import build_dataset + + +class Arguments: + """hardcode values for dataset config""" + def __init__(self, dataset_path): + self.num_classes = 1000 + self.batch_size = 32 + self.input_size = 224 + self.data_set = "IMNET" + self.data_path = dataset_path + +def get_dataloaders_and_eval_func(dataset_path): + """getting dataloader and evaluation function""" + #pylint:disable = unused-variable + args = Arguments(dataset_path=dataset_path) + + dataset_val, _ = build_dataset(is_train=False, args=args) + val_loader = torch.utils.data.DataLoader( + dataset_val, + batch_size=args.batch_size + ) + + train_loader = None + eval_func = evaluate + + return train_loader, val_loader, eval_func + + +def forward_pass(model, kwargs): + """forward pass for compute encodings""" + for idx, (x, _) in enumerate(kwargs['dataloader']): + _ = model(x.to(kwargs['device'])) + if isinstance(kwargs['iterations'], int) and kwargs['iterations'] > 0 and idx >= kwargs['iterations']: + break diff --git a/aimet_zoo_torch/uniformer_classification/evaluators/__init__.py b/aimet_zoo_torch/uniformer_classification/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/uniformer_classification/evaluators/uniformer_classification_quanteval.py b/aimet_zoo_torch/uniformer_classification/evaluators/uniformer_classification_quanteval.py new file mode 100644 index 0000000..9cb1438 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/evaluators/uniformer_classification_quanteval.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" Evaluate Quantization Performance """ + +import argparse +import torch +from aimet_zoo_torch.uniformer_classification import UniformerClassification +from aimet_zoo_torch.common.utils.utils import get_device +from aimet_zoo_torch.uniformer_classification.dataloader.dataloaders_and_eval_func import get_dataloaders_and_eval_func, forward_pass + + +def arguments(raw_args=None): + """ argument parser""" + parser = argparse.ArgumentParser( + description="Evaluation script for PyTorch ImageNet networks." + ) + parser.add_argument( + "--dataset-path", help="Fullpath to ImageNet (ILSVRC2012)", type=str + ) + parser.add_argument( + "--model-config", + help="Select the model configuration", + type=str, + default="uniformer_classification_w4a8", + choices=["uniformer_classification_w4a8", "uniformer_classification_w8a8"], + ) + parser.add_argument( + "--batch-size", help="Data batch size for a model", type=int, default=32 + ) + parser.add_argument( + "--use-cuda", help="Run evaluation on GPU.", type=bool, default=True + ) + args = parser.parse_args(raw_args) + return args + + +def seed(seed_number): + """Set seed for reproducibility""" + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + torch.manual_seed(seed_number) + torch.cuda.manual_seed(seed_number) + torch.cuda.manual_seed_all(seed_number) + + +def main(raw_args=None): + # pylint: disable=too-many-locals + """ main evaluation function""" + seed(0) + args = arguments(raw_args) + device = get_device(args) + iterations = 500 + # pylint: disable = unused-variable + train_loader, val_loader, eval_func = get_dataloaders_and_eval_func(dataset_path=args.dataset_path) + + # Original model + model_orig = UniformerClassification(model_config=args.model_config) + sim_orig = model_orig.get_quantsim(quantized=False) + fp32_orig = model_orig.model + acc_fp32 = eval_func(val_loader, fp32_orig, device=device)["acc1"] + fp_kwargs = {'iterations': iterations, 'dataloader': val_loader, 'device': device} + sim_orig.compute_encodings(forward_pass, forward_pass_callback_args=fp_kwargs) + acc_orig = eval_func(val_loader, sim_orig.model, device=device)["acc1"] + + # Optimized model + model_optim = UniformerClassification(model_config=args.model_config) + sim_optim = model_optim.get_quantsim(quantized=True) + acc_optim = eval_func(val_loader, sim_optim.model, device=device)["acc1"] + + param_bw = model_orig.cfg["optimization_config"]["quantization_configuration"]["param_bw"] + output_bw = model_orig.cfg["optimization_config"]["quantization_configuration"]["output_bw"] + print(f"Original Model | FP32 Environment | Accuracy: {acc_fp32:.4f}") + print(f"Original Model | W{param_bw}A{output_bw} Environment | Accuracy: {acc_orig:.4f}") + print(f"Optimized Model | W{param_bw}A{output_bw} Environment | Accuracy: {acc_optim:.4f}") + + return {"acc_fp32": acc_fp32, "acc_orig": acc_orig, "acc_optim": acc_optim} + +if __name__ == "__main__": + scores_dict = main() + diff --git a/aimet_zoo_torch/uniformer_classification/model/__init__.py b/aimet_zoo_torch/uniformer_classification/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/uniformer_classification/model/image_classification/datasets.py b/aimet_zoo_torch/uniformer_classification/model/image_classification/datasets.py new file mode 100644 index 0000000..f1bf153 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/image_classification/datasets.py @@ -0,0 +1,110 @@ +# Copyright (c) 2015-present, Facebook, Inc. +# All rights reserved. + +import os +import json + +from torchvision import datasets, transforms +from torchvision.datasets.folder import ImageFolder, default_loader + +from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD +from timm.data import create_transform + + +class INatDataset(ImageFolder): + def __init__(self, root, train=True, year=2018, transform=None, target_transform=None, + category='name', loader=default_loader): + self.transform = transform + self.loader = loader + self.target_transform = target_transform + self.year = year + # assert category in ['kingdom','phylum','class','order','supercategory','family','genus','name'] + path_json = os.path.join(root, f'{"train" if train else "val"}{year}.json') + with open(path_json) as json_file: + data = json.load(json_file) + + with open(os.path.join(root, 'categories.json')) as json_file: + data_catg = json.load(json_file) + + path_json_for_targeter = os.path.join(root, f"train{year}.json") + + with open(path_json_for_targeter) as json_file: + data_for_targeter = json.load(json_file) + + targeter = {} + indexer = 0 + for elem in data_for_targeter['annotations']: + king = [] + king.append(data_catg[int(elem['category_id'])][category]) + if king[0] not in targeter.keys(): + targeter[king[0]] = indexer + indexer += 1 + self.nb_classes = len(targeter) + + self.samples = [] + for elem in data['images']: + cut = elem['file_name'].split('/') + target_current = int(cut[2]) + path_current = os.path.join(root, cut[0], cut[2], cut[3]) + + categors = data_catg[target_current] + target_current_true = targeter[categors[category]] + self.samples.append((path_current, target_current_true)) + + # __getitem__ and __len__ inherited from ImageFolder + + +def build_dataset(is_train, args): + transform = build_transform(is_train, args) + + if args.data_set == 'CIFAR': + dataset = datasets.CIFAR100(args.data_path, train=is_train, transform=transform) + nb_classes = 100 + elif args.data_set == 'IMNET': + root = os.path.join(args.data_path, 'train' if is_train else 'val') #(args.data_path, 'ILSVRC2012_img_train' if is_train else 'ILSVRC2012_img_val') + dataset = datasets.ImageFolder(root, transform=transform) + nb_classes = 1000 + elif args.data_set == 'INAT': + dataset = INatDataset(args.data_path, train=is_train, year=2018, + category=args.inat_category, transform=transform) + nb_classes = dataset.nb_classes + elif args.data_set == 'INAT19': + dataset = INatDataset(args.data_path, train=is_train, year=2019, + category=args.inat_category, transform=transform) + nb_classes = dataset.nb_classes + + return dataset, nb_classes + + +def build_transform(is_train, args): + resize_im = args.input_size > 32 + if is_train: + # this should always dispatch to transforms_imagenet_train + transform = create_transform( + input_size=args.input_size, + is_training=True, + color_jitter=args.color_jitter, + auto_augment=args.aa, + interpolation=args.train_interpolation, + re_prob=args.reprob, + re_mode=args.remode, + re_count=args.recount, + ) + if not resize_im: + # replace RandomResizedCropAndInterpolation with + # RandomCrop + transform.transforms[0] = transforms.RandomCrop( + args.input_size, padding=4) + return transform + + t = [] + if resize_im: + size = int((256 / 224) * args.input_size) + t.append( + transforms.Resize(size, interpolation=3), # to maintain same ratio w.r.t. 224 images + ) + t.append(transforms.CenterCrop(args.input_size)) + + t.append(transforms.ToTensor()) + t.append(transforms.Normalize(IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD)) + return transforms.Compose(t) diff --git a/aimet_zoo_torch/uniformer_classification/model/image_classification/elementwise_ops.py b/aimet_zoo_torch/uniformer_classification/model/image_classification/elementwise_ops.py new file mode 100644 index 0000000..c0369e1 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/image_classification/elementwise_ops.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + + +import torch +import torch.nn + + +class FloorDivide(torch.nn.Module): + """ Add module for floor divide """ + # pylint:disable=arguments-differ + @staticmethod + def forward(x: int, y: int) -> int: + """ + Forward-pass routine for floor-divide op + """ + return x // y + +class SoftMax(torch.nn.Module): + """ Add module for softmax """ + # pylint:disable=arguments-differ + @staticmethod + def forward(x: torch.Tensor, dim: int) -> torch.Tensor: + """ + Forward-pass routine for softmax + """ + return x.softmax(dim=dim) \ No newline at end of file diff --git a/aimet_zoo_torch/uniformer_classification/model/image_classification/engine.py b/aimet_zoo_torch/uniformer_classification/model/image_classification/engine.py new file mode 100644 index 0000000..3e7b2a3 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/image_classification/engine.py @@ -0,0 +1,107 @@ +# Copyright (c) 2015-present, Facebook, Inc. +# All rights reserved. + +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +""" +Train and eval functions used in main.py +""" +import math +import sys +from typing import Iterable, Optional + +import torch + +from timm.data import Mixup +from timm.utils import accuracy, ModelEma + +from .losses import DistillationLoss +from . import utils + + +def train_one_epoch(model: torch.nn.Module, criterion: DistillationLoss, + data_loader: Iterable, optimizer: torch.optim.Optimizer, + device: torch.device, epoch: int, loss_scaler, max_norm: float = 0, + model_ema: Optional[ModelEma] = None, mixup_fn: Optional[Mixup] = None, + set_training_mode=True): + model.train(set_training_mode) + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}')) + header = 'Epoch: [{}]'.format(epoch) + print_freq = 10 + + for samples, targets in metric_logger.log_every(data_loader, print_freq, header): + samples = samples.to(device, non_blocking=True) + targets = targets.to(device, non_blocking=True) + + if mixup_fn is not None: + samples, targets = mixup_fn(samples, targets) + + with torch.cuda.amp.autocast(enabled=False): + outputs = model(samples) + loss = criterion(samples, outputs, targets) + + loss_value = loss.item() + + if not math.isfinite(loss_value): + print("Loss is {}, stopping training".format(loss_value)) + sys.exit(1) + + optimizer.zero_grad() + + # this attribute is added by timm on one optimizer (adahessian) + is_second_order = hasattr(optimizer, 'is_second_order') and optimizer.is_second_order + loss_scaler(loss, optimizer, clip_grad=max_norm, + parameters=model.parameters(), create_graph=is_second_order) + + torch.cuda.synchronize() + if model_ema is not None: + model_ema.update(model) + + metric_logger.update(loss=loss_value) + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + return {k: meter.global_avg for k, meter in metric_logger.meters.items()} + + +@torch.no_grad() +def evaluate(data_loader, model, device): + criterion = torch.nn.CrossEntropyLoss() + + metric_logger = utils.MetricLogger(delimiter=" ") + header = 'Test:' + + # switch to evaluation mode + model.eval() + + for images, target in metric_logger.log_every(data_loader, 10, header): + images = images.to(device, non_blocking=True) + target = target.to(device, non_blocking=True) + + # compute output + with torch.cuda.amp.autocast(enabled=False): + output = model(images) + loss = criterion(output, target) + + acc1, acc5 = accuracy(output, target, topk=(1, 5)) + + batch_size = images.shape[0] + metric_logger.update(loss=loss.item()) + metric_logger.meters['acc1'].update(acc1.item(), n=batch_size) + metric_logger.meters['acc5'].update(acc5.item(), n=batch_size) + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print('* Acc@1 {top1.global_avg:.3f} Acc@5 {top5.global_avg:.3f} loss {losses.global_avg:.3f}' + .format(top1=metric_logger.acc1, top5=metric_logger.acc5, losses=metric_logger.loss)) + + return {k: meter.global_avg for k, meter in metric_logger.meters.items()} diff --git a/aimet_zoo_torch/uniformer_classification/model/image_classification/generate_tensorboard.py b/aimet_zoo_torch/uniformer_classification/model/image_classification/generate_tensorboard.py new file mode 100644 index 0000000..6fb83fd --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/image_classification/generate_tensorboard.py @@ -0,0 +1,28 @@ +import os +import json +from tensorboardX import SummaryWriter + +exp_path_list = ['exp'] +log_keys = ['train_lr', 'train_loss', 'test_loss', 'test_acc1', 'test_acc5'] +ignore_exp = [] + +for path in exp_path_list: + for exp in os.listdir(path): + log_path = os.path.join('.', path, exp, 'ckpt', 'log.txt') + if os.path.exists(log_path): + tensorboard_path = os.path.join('.', path, exp, 'events') + if os.path.exists(tensorboard_path): + for old_exp in os.listdir(tensorboard_path): + delete_path = os.path.join(tensorboard_path, old_exp) + print('delete:', delete_path) + os.remove(delete_path) + tb_logger = SummaryWriter(tensorboard_path) + if exp not in ignore_exp: + with open(log_path, 'r') as f: + lines = f.readlines() + for line in lines: + log = json.loads(line.rstrip()) + for k in log_keys: + tb_logger.add_scalar(k, log[k], log['epoch']) + print("load ok in:", tensorboard_path) + tb_logger.close() diff --git a/aimet_zoo_torch/uniformer_classification/model/image_classification/losses.py b/aimet_zoo_torch/uniformer_classification/model/image_classification/losses.py new file mode 100644 index 0000000..a56dde4 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/image_classification/losses.py @@ -0,0 +1,65 @@ +# Copyright (c) 2015-present, Facebook, Inc. +# All rights reserved. +""" +Implements the knowledge distillation loss +""" +import torch +from torch.nn import functional as F + + +class DistillationLoss(torch.nn.Module): + """ + This module wraps a standard criterion and adds an extra knowledge distillation loss by + taking a teacher model prediction and using it as additional supervision. + """ + + def __init__(self, base_criterion: torch.nn.Module, teacher_model: torch.nn.Module, + distillation_type: str, alpha: float, tau: float): + super().__init__() + self.base_criterion = base_criterion + self.teacher_model = teacher_model + assert distillation_type in ['none', 'soft', 'hard'] + self.distillation_type = distillation_type + self.alpha = alpha + self.tau = tau + + def forward(self, inputs, outputs, labels): + """ + Args: + inputs: The original inputs that are feed to the teacher model + outputs: the outputs of the model to be trained. It is expected to be + either a Tensor, or a Tuple[Tensor, Tensor], with the original output + in the first position and the distillation predictions as the second output + labels: the labels for the base criterion + """ + outputs_kd = None + if not isinstance(outputs, torch.Tensor): + # assume that the model outputs a tuple of [outputs, outputs_kd] + outputs, outputs_kd = outputs + base_loss = self.base_criterion(outputs, labels) + if self.distillation_type == 'none': + return base_loss + + if outputs_kd is None: + raise ValueError("When knowledge distillation is enabled, the model is " + "expected to return a Tuple[Tensor, Tensor] with the output of the " + "class_token and the dist_token") + # don't backprop throught the teacher + with torch.no_grad(): + teacher_outputs = self.teacher_model(inputs) + + if self.distillation_type == 'soft': + T = self.tau + # taken from https://github.com/peterliht/knowledge-distillation-pytorch/blob/master/model/net.py#L100 + # with slight modifications + distillation_loss = F.kl_div( + F.log_softmax(outputs_kd / T, dim=1), + F.log_softmax(teacher_outputs / T, dim=1), + reduction='sum', + log_target=True + ) * (T * T) / outputs_kd.numel() + elif self.distillation_type == 'hard': + distillation_loss = F.cross_entropy(outputs_kd, teacher_outputs.argmax(dim=1)) + + loss = base_loss * (1 - self.alpha) + distillation_loss * self.alpha + return loss diff --git a/aimet_zoo_torch/uniformer_classification/model/image_classification/models/__init__.py b/aimet_zoo_torch/uniformer_classification/model/image_classification/models/__init__.py new file mode 100644 index 0000000..72c1e6b --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/image_classification/models/__init__.py @@ -0,0 +1 @@ +from .uniformer import * \ No newline at end of file diff --git a/aimet_zoo_torch/uniformer_classification/model/image_classification/models/uniformer.py b/aimet_zoo_torch/uniformer_classification/model/image_classification/models/uniformer.py new file mode 100644 index 0000000..2d8183c --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/image_classification/models/uniformer.py @@ -0,0 +1,428 @@ +# Copyright (c) 2015-present, Facebook, Inc. +# All rights reserved. + +# ----------------------------------------------------------------------- +# Copyright 2022 SenseTime X-Lab. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ----------------------------------------------------------------------- + +from collections import OrderedDict +import torch +import torch.nn as nn +from functools import partial +import torch.nn.functional as F +import math +from timm.models.vision_transformer import _cfg +from timm.models.registry import register_model +from timm.models.layers import trunc_normal_, DropPath, to_2tuple +from aimet_torch import elementwise_ops +from ..elementwise_ops import FloorDivide, SoftMax + +layer_scale = False +init_value = 1e-6 + + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class CMlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Conv2d(in_features, hidden_features, 1) + self.act = act_layer() + self.fc2 = nn.Conv2d(hidden_features, out_features, 1) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + self.floor_divide1 = FloorDivide() + self.floor_divide2 = FloorDivide() + head_dim = self.floor_divide1(dim, num_heads) + # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights + self.scale = qk_scale or head_dim ** -0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + self.matmul1 = elementwise_ops.MatMul() + self.matmul2 = elementwise_ops.MatMul() + + self.mul1 = elementwise_ops.Multiply() + self.softmax = SoftMax() + + def forward(self, x): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + attn = self.mul1((self.matmul1(q, k.transpose(-2, -1))), self.scale) + attn = self.softmax(attn, -1) + attn = self.attn_drop(attn) + + x = (self.matmul2(attn, v)).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class CBlock(nn.Module): + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.pos_embed = nn.Conv2d(dim, dim, 3, padding=1, groups=dim) + self.norm1 = nn.BatchNorm2d(dim) + self.conv1 = nn.Conv2d(dim, dim, 1) + self.conv2 = nn.Conv2d(dim, dim, 1) + self.attn = nn.Conv2d(dim, dim, 5, padding=2, groups=dim) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = nn.BatchNorm2d(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = CMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + self.add1 = elementwise_ops.Add() + self.add2 = elementwise_ops.Add() + self.add3 = elementwise_ops.Add() + + def forward(self, x): + x = self.add1(x, self.pos_embed(x)) + x = self.add2(x, self.drop_path(self.conv2(self.attn(self.conv1(self.norm1(x)))))) + x = self.add3(x, self.drop_path(self.mlp(self.norm2(x)))) + return x + + +class SABlock(nn.Module): + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.pos_embed = nn.Conv2d(dim, dim, 3, padding=1, groups=dim) + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, + attn_drop=attn_drop, proj_drop=drop) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + global layer_scale + self.ls = layer_scale + if self.ls: + global init_value + print(f"Use layer_scale: {layer_scale}, init_values: {init_value}") + self.gamma_1 = nn.Parameter(init_value * torch.ones((dim)),requires_grad=True) + self.gamma_2 = nn.Parameter(init_value * torch.ones((dim)),requires_grad=True) + + self.add1 = elementwise_ops.Add() + self.add2 = elementwise_ops.Add() + self.add3 = elementwise_ops.Add() + self.add4 = elementwise_ops.Add() + self.add5 = elementwise_ops.Add() + + def forward(self, x): + x = self.add1(x, self.pos_embed(x)) + B, N, H, W = x.shape + x = x.flatten(2).transpose(1, 2) + if self.ls: + x = self.add2(x, self.drop_path(self.gamma_1 * self.attn(self.norm1(x)))) + x = self.add3(x, self.drop_path(self.gamma_2 * self.mlp(self.norm2(x)))) + else: + x = self.add4(x, self.drop_path(self.attn(self.norm1(x)))) + x = self.add5(x, self.drop_path(self.mlp(self.norm2(x)))) + x = x.transpose(1, 2).reshape(B, N, H, W) + return x + + +class head_embedding(nn.Module): + def __init__(self, in_channels, out_channels): + super(head_embedding, self).__init__() + + self.floor_divide1 = FloorDivide() + self.floor_divide2 = FloorDivide() + self.floor_divide3 = FloorDivide() + + self.proj = nn.Sequential( + nn.Conv2d(in_channels, self.floor_divide1(out_channels, 2), kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)), + nn.BatchNorm2d(self.floor_divide2(out_channels, 2)), + nn.GELU(), + nn.Conv2d(self.floor_divide3(out_channels, 2), out_channels, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)), + nn.BatchNorm2d(out_channels), + ) + + def forward(self, x): + x = self.proj(x) + return x + + +class middle_embedding(nn.Module): + def __init__(self, in_channels, out_channels): + super(middle_embedding, self).__init__() + + self.proj = nn.Sequential( + nn.Conv2d(in_channels, out_channels, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)), + nn.BatchNorm2d(out_channels), + ) + + def forward(self, x): + x = self.proj(x) + return x + + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + """ + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + self.floor_divide1 = FloorDivide() + self.floor_divide2 = FloorDivide() + num_patches = (self.floor_divide1(img_size[1], patch_size[1])) * (self.floor_divide2(img_size[0], patch_size[0])) + self.img_size = img_size + self.patch_size = patch_size + self.num_patches = num_patches + self.norm = nn.LayerNorm(embed_dim) + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + + def forward(self, x): + B, C, H, W = x.shape + # FIXME look at relaxing size constraints + #assert H == self.img_size[0] and W == self.img_size[1], \ + # f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." + x = self.proj(x) + B, C, H, W = x.shape + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous() + return x + + +class UniFormer(nn.Module): + """ Vision Transformer + A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - + https://arxiv.org/abs/2010.11929 + """ + def __init__(self, depth=[3, 4, 8, 3], img_size=224, in_chans=3, num_classes=1000, embed_dim=[64, 128, 320, 512], + head_dim=64, mlp_ratio=4., qkv_bias=True, qk_scale=None, representation_size=None, + drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=None, conv_stem=False): + """ + Args: + depth (list): depth of each stage + img_size (int, tuple): input image size + in_chans (int): number of input channels + num_classes (int): number of classes for classification head + embed_dim (list): embedding dimension of each stage + head_dim (int): head dimension + mlp_ratio (int): ratio of mlp hidden dim to embedding dim + qkv_bias (bool): enable bias for qkv if True + qk_scale (float): override default qk scale of head_dim ** -0.5 if set + representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set + drop_rate (float): dropout rate + attn_drop_rate (float): attention dropout rate + drop_path_rate (float): stochastic depth rate + norm_layer (nn.Module): normalization layer + conv_stem (bool): whether use overlapped patch stem + """ + super().__init__() + self.num_classes = num_classes + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + + self.floor_divide1 = FloorDivide() + self.floor_divide2 = FloorDivide() + self.floor_divide3 = FloorDivide() + + if conv_stem: + self.patch_embed1 = head_embedding(in_channels=in_chans, out_channels=embed_dim[0]) + self.patch_embed2 = middle_embedding(in_channels=embed_dim[0], out_channels=embed_dim[1]) + self.patch_embed3 = middle_embedding(in_channels=embed_dim[1], out_channels=embed_dim[2]) + self.patch_embed4 = middle_embedding(in_channels=embed_dim[2], out_channels=embed_dim[3]) + else: + self.patch_embed1 = PatchEmbed( + img_size=img_size, patch_size=4, in_chans=in_chans, embed_dim=embed_dim[0]) + self.patch_embed2 = PatchEmbed( + img_size=self.floor_divide1(img_size, 4), patch_size=2, in_chans=embed_dim[0], embed_dim=embed_dim[1]) + self.patch_embed3 = PatchEmbed( + img_size=self.floor_divide2(img_size, 8), patch_size=2, in_chans=embed_dim[1], embed_dim=embed_dim[2]) + self.patch_embed4 = PatchEmbed( + img_size=self.floor_divide3(img_size, 16), patch_size=2, in_chans=embed_dim[2], embed_dim=embed_dim[3]) + + self.pos_drop = nn.Dropout(p=drop_rate) + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depth))] # stochastic depth decay rule + self.floor_divide_modulelist = nn.ModuleList([FloorDivide() for i in range(len(embed_dim))]) + num_heads = [curr_floor_divide(dim, head_dim) for (curr_floor_divide,dim) in zip(self.floor_divide_modulelist,embed_dim)] + self.blocks1 = nn.ModuleList([ + CBlock( + dim=embed_dim[0], num_heads=num_heads[0], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer) + for i in range(depth[0])]) + self.blocks2 = nn.ModuleList([ + CBlock( + dim=embed_dim[1], num_heads=num_heads[1], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+depth[0]], norm_layer=norm_layer) + for i in range(depth[1])]) + self.blocks3 = nn.ModuleList([ + SABlock( + dim=embed_dim[2], num_heads=num_heads[2], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+depth[0]+depth[1]], norm_layer=norm_layer) + for i in range(depth[2])]) + self.blocks4 = nn.ModuleList([ + SABlock( + dim=embed_dim[3], num_heads=num_heads[3], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+depth[0]+depth[1]+depth[2]], norm_layer=norm_layer) + for i in range(depth[3])]) + self.norm = nn.BatchNorm2d(embed_dim[-1]) + + # Representation layer + if representation_size: + self.num_features = representation_size + self.pre_logits = nn.Sequential(OrderedDict([ + ('fc', nn.Linear(embed_dim, representation_size)), + ('act', nn.Tanh()) + ])) + else: + self.pre_logits = nn.Identity() + + # Classifier head + self.head = nn.Linear(embed_dim[-1], num_classes) if num_classes > 0 else nn.Identity() + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @torch.jit.ignore + def no_weight_decay(self): + return {'pos_embed', 'cls_token'} + + def get_classifier(self): + return self.head + + def reset_classifier(self, num_classes, global_pool=''): + self.num_classes = num_classes + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + def forward_features(self, x): + x = self.patch_embed1(x) + x = self.pos_drop(x) + for blk in self.blocks1: + x = blk(x) + x = self.patch_embed2(x) + for blk in self.blocks2: + x = blk(x) + x = self.patch_embed3(x) + for blk in self.blocks3: + x = blk(x) + x = self.patch_embed4(x) + for blk in self.blocks4: + x = blk(x) + x = self.norm(x) + x = self.pre_logits(x) + return x + + def forward(self, x): + x = self.forward_features(x) + x = x.flatten(2).mean(-1) + x = self.head(x) + return x + + +@register_model +def uniformer_small(pretrained=True, **kwargs): + model = UniFormer( + depth=[3, 4, 8, 3], + embed_dim=[64, 128, 320, 512], head_dim=64, mlp_ratio=4, qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs) + model.default_cfg = _cfg() + return model + + +@register_model +def uniformer_small_plus(pretrained=True, **kwargs): + model = UniFormer( + depth=[3, 5, 9, 3], conv_stem=True, + embed_dim=[64, 128, 320, 512], head_dim=32, mlp_ratio=4, qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs) + model.default_cfg = _cfg() + return model + + +@register_model +def uniformer_small_plus_dim64(pretrained=True, **kwargs): + model = UniFormer( + depth=[3, 5, 9, 3], conv_stem=True, + embed_dim=[64, 128, 320, 512], head_dim=64, mlp_ratio=4, qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs) + model.default_cfg = _cfg() + return model + + + +@register_model +def uniformer_base(pretrained=True, **kwargs): + model = UniFormer( + depth=[5, 8, 20, 7], + embed_dim=[64, 128, 320, 512], head_dim=64, mlp_ratio=4, qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs) + model.default_cfg = _cfg() + return model + + +@register_model +def uniformer_base_ls(pretrained=True, **kwargs): + global layer_scale + layer_scale = True + model = UniFormer( + depth=[5, 8, 20, 7], + embed_dim=[64, 128, 320, 512], head_dim=64, mlp_ratio=4, qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs) + model.default_cfg = _cfg() + return model diff --git a/aimet_zoo_torch/uniformer_classification/model/image_classification/no_scaling_scaler.py b/aimet_zoo_torch/uniformer_classification/model/image_classification/no_scaling_scaler.py new file mode 100644 index 0000000..526eaea --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/image_classification/no_scaling_scaler.py @@ -0,0 +1,31 @@ +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# Changes from QuIC are licensed under the terms and conditions at +# https://github.com/quic/aimet-model-zoo/blob/develop/LICENSE.pdf +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= + +import torch + +class NoScalingScaler: + state_dict_key = "amp_scaler" #"noscaling_scaler" + + def __init__(self): + self._scaler = torch.cuda.amp.GradScaler() + + def __call__(self, loss, optimizer, clip_grad=None, clip_mode='norm', parameters=None, create_graph=False): + loss.backward(create_graph=create_graph) + if clip_grad is not None: + assert parameters is not None + dispatch_clip_grad(parameters, clip_grad, mode=clip_mode) + optimizer.step() + + + def state_dict(self): + return self._scaler.state_dict() + + def load_state_dict(self, state_dict): + self._scaler.load_state_dict(state_dict) diff --git a/aimet_zoo_torch/uniformer_classification/model/image_classification/samplers.py b/aimet_zoo_torch/uniformer_classification/model/image_classification/samplers.py new file mode 100644 index 0000000..61e50e6 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/image_classification/samplers.py @@ -0,0 +1,60 @@ +# modified from https://github.com/facebookresearch/deit + +import torch +import torch.distributed as dist +import math + + +class RASampler(torch.utils.data.Sampler): + """Sampler that restricts data loading to a subset of the dataset for distributed, + with repeated augmentation. + It ensures that different each augmented version of a sample will be visible to a + different process (GPU) + Heavily based on torch.utils.data.DistributedSampler + """ + + def __init__(self, dataset, num_replicas=None, rank=None, shuffle=True): + if num_replicas is None: + if not dist.is_available(): + raise RuntimeError( + "Requires distributed package to be available") + num_replicas = dist.get_world_size() + if rank is None: + if not dist.is_available(): + raise RuntimeError("Requires distributed package to be available") + rank = dist.get_rank() + self.dataset = dataset + self.num_replicas = num_replicas + self.rank = rank + self.epoch = 0 + self.num_samples = int(math.ceil(len(self.dataset) * 3.0 / self.num_replicas)) + self.total_size = self.num_samples * self.num_replicas + # self.num_selected_samples = int(math.ceil(len(self.dataset) / self.num_replicas)) + self.num_selected_samples = int(math.floor(len(self.dataset) // 256 * 256 / self.num_replicas)) + self.shuffle = shuffle + + def __iter__(self): + # deterministically shuffle based on epoch + g = torch.Generator() + g.manual_seed(self.epoch) + if self.shuffle: + indices = torch.randperm(len(self.dataset), generator=g).tolist() + else: + indices = list(range(len(self.dataset))) + + # add extra samples to make it evenly divisible + indices = [ele for ele in indices for i in range(3)] + indices += indices[:(self.total_size - len(indices))] + assert len(indices) == self.total_size + + # subsample + indices = indices[self.rank:self.total_size:self.num_replicas] + assert len(indices) == self.num_samples + + return iter(indices[:self.num_selected_samples]) + + def __len__(self): + return self.num_selected_samples + + def set_epoch(self, epoch): + self.epoch = epoch diff --git a/aimet_zoo_torch/uniformer_classification/model/image_classification/utils.py b/aimet_zoo_torch/uniformer_classification/model/image_classification/utils.py new file mode 100644 index 0000000..211e762 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/image_classification/utils.py @@ -0,0 +1,262 @@ +# modified from https://github.com/facebookresearch/LeViT + +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" +import io +import os +import time +from collections import defaultdict, deque +import datetime + +import torch +import torch.distributed as dist + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], + dtype=torch.float64, device='cuda') + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value) + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format( + type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append( + "{}: {}".format(name, str(meter)) + ) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None): + i = 0 + if not header: + header = '' + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt='{avg:.4f}') + data_time = SmoothedValue(fmt='{avg:.4f}') + space_fmt = ':' + str(len(str(len(iterable)))) + 'd' + log_msg = [ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}' + ] + if torch.cuda.is_available(): + log_msg.append('max mem: {memory:.0f}') + log_msg = self.delimiter.join(log_msg) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB)) + else: + print(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time))) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('{} Total time: {} ({:.4f} s / it)'.format( + header, total_time_str, total_time / len(iterable))) + + +def _load_checkpoint_for_ema(model_ema, checkpoint): + """ + Workaround for ModelEma._load_checkpoint to accept an already-loaded object + """ + mem_file = io.BytesIO() + torch.save(checkpoint, mem_file) + mem_file.seek(0) + model_ema._load_checkpoint(mem_file) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop('force', False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ['WORLD_SIZE']) + args.gpu = int(os.environ['LOCAL_RANK']) + elif 'SLURM_PROCID' in os.environ: + args.rank = int(os.environ['SLURM_PROCID']) + args.gpu = args.rank % torch.cuda.device_count() + else: + print('Not using distributed mode') + args.distributed = False + return + + args.distributed = True + + print("args.gpu ==== ", args.gpu) + torch.cuda.set_device(args.gpu) + args.dist_backend = 'nccl' + print('| distributed init (rank {}): {}'.format( + args.rank, args.dist_url), flush=True) + torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +def replace_batchnorm(net): + for child_name, child in net.named_children(): + if hasattr(child, 'fuse'): + setattr(net, child_name, child.fuse()) + elif isinstance(child, torch.nn.Conv2d): + child.bias = torch.nn.Parameter(torch.zeros(child.weight.size(0))) + elif isinstance(child, torch.nn.BatchNorm2d): + setattr(net, child_name, torch.nn.Identity()) + else: + replace_batchnorm(child) + + +def replace_layernorm(net): + import apex + for child_name, child in net.named_children(): + if isinstance(child, torch.nn.LayerNorm): + setattr(net, child_name, apex.normalization.FusedLayerNorm( + child.weight.size(0))) + else: + replace_layernorm(child) diff --git a/aimet_zoo_torch/uniformer_classification/model/model_cards/__init__.py b/aimet_zoo_torch/uniformer_classification/model/model_cards/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aimet_zoo_torch/uniformer_classification/model/model_cards/uniformer_classification_w8a8.json b/aimet_zoo_torch/uniformer_classification/model/model_cards/uniformer_classification_w8a8.json new file mode 100644 index 0000000..298e096 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/model_cards/uniformer_classification_w8a8.json @@ -0,0 +1,26 @@ +{ + "name": "Uniformer Small", + "framework": "pytorch", + "task": "classification", + "model_args": { + "num_classes": 1000 + }, + "input_shape": [null, 3, 224, 224], + "trainig_dataset": "Imagenet", + "optimization_config": { + "quantization_configuration": + { + "param_bw": 8, + "output_bw": 8, + "input_quantization": true, + "quant_scheme": "tf_enhanced", + "techniques": ["qat", "bn_fold"] + } + }, + "artifacts": { + "url_pre_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_uniformer_classification/uniformer_classification_fp32.pth", + "url_post_opt_weights": "https://github.com/quic/aimet-model-zoo/releases/download/torch_uniformer_classification/uniformer_classification_w8a8.pth", + "url_aimet_encodings": "https://github.com/quic/aimet-model-zoo/releases/download/torch_uniformer_classification/uniformer_classification_w8a8.encodings", + "url_aimet_config": "https://raw.githubusercontent.com/quic/aimet/release-aimet-1.22.1/TrainingExtensions/common/src/python/aimet_common/quantsim_config/default_config_per_channel.json" + } +} diff --git a/aimet_zoo_torch/uniformer_classification/model/model_definition.py b/aimet_zoo_torch/uniformer_classification/model/model_definition.py new file mode 100644 index 0000000..76a41da --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/model/model_definition.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# ============================================================================= +# @@-COPYRIGHT-START-@@ +# +# Copyright (c) 2023 of Qualcomm Innovation Center, Inc. All rights reserved. +# +# @@-COPYRIGHT-END-@@ +# ============================================================================= +"""Class for downloading and setting up model for AIMET model zoo""" +# pylint:disable = import-error +import json +import os +import pathlib +import torch +from aimet_common.defs import QuantScheme +from aimet_torch.cross_layer_equalization import equalize_model +from aimet_torch.quantsim import QuantizationSimModel, load_encodings_to_sim +from aimet_zoo_torch.common.downloader import Downloader +from aimet_zoo_torch.uniformer_classification.model.image_classification.models.uniformer import uniformer_small + + +class UniformerClassification(Downloader): + """Uniformer Classification parent class with automated loading of weights and providing a QuantSim with pre-computed encodings""" + def __init__(self, model_config=None): + parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) + self.cfg = False + if model_config: + config_filepath = parent_dir + "/model_cards/" + model_config + ".json" + with open(config_filepath) as f_in: + self.cfg = json.load(f_in) + if model_config: + Downloader.__init__( + self, + url_pre_opt_weights=self.cfg["artifacts"]["url_pre_opt_weights"], + url_post_opt_weights=self.cfg["artifacts"]["url_post_opt_weights"], + url_aimet_encodings=self.cfg["artifacts"]["url_aimet_encodings"], + url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], + model_dir=parent_dir, + model_config=model_config, + ) + self.model = uniformer_small() + self.input_shape = tuple(x if x is not None else 1 for x in self.cfg["input_shape"]) + self.model_config = model_config + + def from_pretrained(self, quantized=False): + """get pretrained model from downloading or coping""" + if not self.cfg: + raise NotImplementedError( + "There are no pretrained weights available for the model_config passed" + ) + self._download_pre_opt_weights() + self._download_post_opt_weights() + self._download_aimet_config() + self._download_aimet_encodings() + if quantized: + equalize_model(self.model, self.input_shape) + state_dict = torch.load(self.path_post_opt_weights) + self.model.load_state_dict(state_dict) + del state_dict + else: + state_dict = torch.load(self.path_pre_opt_weights) + self.model.load_state_dict(state_dict) + del state_dict + self.model.eval() + self.model.cuda() + + def get_quantsim(self, quantized=False): + """" to get quantsim object for model from loading/computing proper encodings""" + if quantized: + self.from_pretrained(quantized=True) + else: + self.from_pretrained(quantized=False) + device = torch.device("cuda") + dummy_input = torch.rand(self.input_shape, device=device) + quant_config = self.cfg["optimization_config"]["quantization_configuration"] + kwargs = { + "quant_scheme": QuantScheme.training_range_learning_with_tf_init if quantized else 'tf', + "default_param_bw": quant_config["param_bw"], + "default_output_bw": quant_config["output_bw"], + "config_file": self.path_aimet_config, + "dummy_input": dummy_input, + } + sim = QuantizationSimModel(self.model.cuda(), **kwargs) + sim.compute_encodings(lambda m, _: m(dummy_input), None) + if self.path_aimet_encodings and quantized: + load_encodings_to_sim(sim, self.path_aimet_encodings) + print("Loaded encodings!") + if self.path_adaround_encodings and quantized: + sim.set_and_freeze_param_encodings(self.path_adaround_encodings) + return sim diff --git a/aimet_zoo_torch/uniformer_classification/requirements.txt b/aimet_zoo_torch/uniformer_classification/requirements.txt new file mode 100644 index 0000000..28bf269 --- /dev/null +++ b/aimet_zoo_torch/uniformer_classification/requirements.txt @@ -0,0 +1,3 @@ +torch>=1.9.0 +torchvision>=0.8.1 +timm>=0.4.12 diff --git a/aimet_zoo_torch/vit/ViT.md b/aimet_zoo_torch/vit/ViT.md index e94e2e1..03573f5 100644 --- a/aimet_zoo_torch/vit/ViT.md +++ b/aimet_zoo_torch/vit/ViT.md @@ -2,11 +2,11 @@ This document describes evaluation of optimized checkpoints for vision transformer (ViT) for image classification ## AIMET installation and setup -Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.24/packaging/install.md) (*Torch GPU* variant) before proceeding further. +Please [install and setup AIMET](https://github.com/quic/aimet/blob/release-aimet-1.26/packaging/install.md) (*Torch GPU* variant) before proceeding further. **NOTE** - All AIMET releases are available here: https://github.com/quic/aimet/releases -- This model has been tested using AIMET version *1.24.0* (i.e. set `release_tag="1.24.0"` in the above instructions). +- This model has been tested using AIMET version *1.24.0* (i.e. set `release_tag="1.26.0"` in the above instructions). - This model is compatible with the PyTorch GPU variant of AIMET (i.e. set `AIMET_VARIANT="torch_gpu"` in the above instructions). ## Additional Setup Dependencies diff --git a/aimet_zoo_torch/vit/dataloader/dataloaders.py b/aimet_zoo_torch/vit/dataloader/dataloaders.py index 6b07889..99c6b8f 100644 --- a/aimet_zoo_torch/vit/dataloader/dataloaders.py +++ b/aimet_zoo_torch/vit/dataloader/dataloaders.py @@ -9,8 +9,9 @@ # ============================================================================= #pylint: skip-file """ module for getting dataloders""" - +import os from PIL import Image +import pathlib from datasets import load_dataset import torch from torch.utils.data import DataLoader @@ -24,13 +25,29 @@ ToTensor, ) -def get_dataloaders(config,feature_extractor,interpolate=False): +# pylint: disable-msg=R0902 +class DataConfig: + """adding hardcoded values into args from parseargs() and return config object""" + + def __init__(self, args): + self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent.parent) + self.dataset_name = os.path.join(self.parent_dir,"dataloader/utils/imagenet.py") + self.max_eval_samples = None + self.max_train_samples = None + self.clamp_quantizer = False + self.per_device_train_batch_size = 8 + self.image_normalization = True + for arg in vars(args): + setattr(self, arg, getattr(args, arg)) + +def get_dataloaders(args,feature_extractor,interpolate=False): """Get train_dataloader and val_dataloader """ - # get dataset from config - - dataset = get_dataset(config) + # hardcoded values for args + args = DataConfig(args) + # get dataset from args + dataset = get_dataset(args) # Prepare label mappings. # We'll include these in the model's config to get human readable labels @@ -54,7 +71,7 @@ def get_dataloaders(config,feature_extractor,interpolate=False): CenterCrop(feature_extractor.size), ToTensor(), ] - if config.image_normalization: + if args.image_normalization: _train_transforms.append(normalize) _val_transforms.append(normalize) train_transforms = Compose(_train_transforms) @@ -86,19 +103,19 @@ def preprocess_val(example_batch): image.convert("RGB")) for image in example_batch["image"]] return example_batch - if config.max_train_samples is not None: + if args.max_train_samples is not None: dataset["train"] = ( dataset["train"] - .shuffle(seed=config.seed) - .select(range(config.max_train_samples)) + .shuffle(seed=args.seed) + .select(range(args.max_train_samples)) ) # Set the training transforms train_dataset = dataset["train"].with_transform(preprocess_train) - if config.max_eval_samples is not None: + if args.max_eval_samples is not None: dataset["validation"] = ( dataset["validation"] - .shuffle(seed=config.seed) - .select(range(config.max_eval_samples)) + .shuffle(seed=args.seed) + .select(range(args.max_eval_samples)) ) # Set the validation transforms eval_dataset = dataset["validation"].with_transform(preprocess_val) @@ -121,12 +138,12 @@ def collate_fn(examples): train_dataset, shuffle=True, collate_fn=collate_fn, - batch_size=config.per_device_train_batch_size, + batch_size=args.per_device_train_batch_size, ) eval_dataloader = DataLoader( eval_dataset, collate_fn=collate_fn, - batch_size=config.per_device_eval_batch_size, + batch_size=args.per_device_eval_batch_size, ) def eval_function(model,args): @@ -160,10 +177,10 @@ def eval_function(model,args): return train_dataloader,eval_dataloader,eval_function -def get_dataset(config): +def get_dataset(args): """get imagenet dataset Parameters: - config: location of imagenet train and validation dataset + args: location of imagenet train and validation dataset Returns: dataset: imagenet dataset """ @@ -174,11 +191,14 @@ def get_dataset(config): # In distributed training, the load_dataset function guarantees that only one local process can concurrently # download the dataset. # imagenet custom script loader + + # hardcoded values for args + args = DataConfig(args) data_files = {} - data_files["train"] = config.train_dir - # if config.validation_dir is not None: - data_files["validation"] = config.validation_dir - # if config.dataset_name.endswith(".py"): - dataset = load_dataset(config.dataset_name, data_dir=data_files) + data_files["train"] = args.train_dir + # if args.validation_dir is not None: + data_files["validation"] = args.validation_dir + # if args.dataset_name.endswith(".py"): + dataset = load_dataset(args.dataset_name, data_dir=data_files) return dataset diff --git a/aimet_zoo_torch/vit/evaluators/vit_quanteval.py b/aimet_zoo_torch/vit/evaluators/vit_quanteval.py index 103ecd7..d2e1cf9 100644 --- a/aimet_zoo_torch/vit/evaluators/vit_quanteval.py +++ b/aimet_zoo_torch/vit/evaluators/vit_quanteval.py @@ -18,7 +18,7 @@ from accelerate import Accelerator from accelerate.logging import get_logger from accelerate.utils import set_seed -from aimet_zoo_torch.vit.dataloader import get_dataloaders, get_dataset +from aimet_zoo_torch.vit.dataloader import get_dataloaders,get_dataset from aimet_zoo_torch.vit import vit logger = get_logger(__name__) @@ -30,7 +30,7 @@ ) -def parse_args(): +def parse_args(raw_args): """argument parser""" parser = argparse.ArgumentParser( description="Evaluating VIT/MobileVIT Transformers model on an imagenet dataset" @@ -58,33 +58,22 @@ def parse_args(): default=8, help="Batch size (per device) for the evaluation dataloader.", ) - args = parser.parse_args() + parser.add_argument( + "--seed", + type=int, + default=2022, + help="training seed", + ) + args = parser.parse_args(raw_args) for arg in vars(args): print("{:30s} : {}".format(arg, getattr(args, arg))) return args -# pylint: disable-msg=R0902 -class DataConfig: - """adding hardcoded values into args from parseargs() and return config object""" - - def __init__(self, args): - self.dataset_name = "../dataloader/utils/imagenet.py" - self.seed = 2022 - self.max_eval_samples = None - self.max_train_samples = None - self.clamp_quantizer = False - self.per_device_train_batch_size = 8 - self.image_normalization = True - for arg in vars(args): - setattr(self, arg, getattr(args, arg)) - - -def main(): +def main(raw_args=None): """Evaluation main function""" - args = parse_args() - config = DataConfig(args) + args = parse_args(raw_args) # Initialize the accelerator. We will let the accelerator handle device placement for us in this example. # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers # in the environment @@ -105,13 +94,12 @@ def main(): transformers.utils.logging.set_verbosity_error() # If passed along, set the training seed now. - if config.seed is not None: - set_seed(config.seed) + if args.seed is not None: + set_seed(args.seed) accelerator.wait_for_everyone() # get dataset for gathering information to load model - dataset = get_dataset(config) - + dataset = get_dataset(args) # loading finetuned original model model = vit(model_config=args.model_config, quantized=False) model_orig = model.get_model_from_pretrained(dataset) @@ -120,7 +108,7 @@ def main(): # load modularized eval_function and dataloaders train_dataloader, eval_dataloader, eval_function = get_dataloaders( - config, feature_extractor + args, feature_extractor ) # Prepare everything with our `accelerator`. diff --git a/aimet_zoo_torch/vit/model/model_cards/vit_w8a8.json b/aimet_zoo_torch/vit/model/model_cards/vit_w8a8.json index 5c4de72..e0e7ea0 100644 --- a/aimet_zoo_torch/vit/model/model_cards/vit_w8a8.json +++ b/aimet_zoo_torch/vit/model/model_cards/vit_w8a8.json @@ -3,12 +3,12 @@ "framework": "pytorch", "task": "", "model_args": { - "quantized":{"model_name_or_path": "../model/weights/downloaded_weights"}, - "original":{"model_name_or_path": "google/vit-base-patch16-224"}, - "dataset_name": "../model/utils/imagenet.py", + "quantized":{"model_name_or_path": "weights/downloaded_weights"}, + "original":{"model_name_or_path": "google/vit-base-patch16-224"}, + "dataset_name": "utils/imagenet.py", "higher_resolution": "False", "ignore_mismatched_sizes": "False", - "config_file": "../model/weights/aimet_config", + "config_file": "weights/aimet_config", "clamp_quantizer": "False", "clamping_value": "30.0" }, diff --git a/aimet_zoo_torch/vit/model/model_definition.py b/aimet_zoo_torch/vit/model/model_definition.py index 352c333..1986727 100644 --- a/aimet_zoo_torch/vit/model/model_definition.py +++ b/aimet_zoo_torch/vit/model/model_definition.py @@ -13,6 +13,7 @@ import os import csv from collections import defaultdict +import pathlib import torch from transformers import AutoConfig as Config from transformers import AutoFeatureExtractor as FeatureExtractor @@ -36,25 +37,38 @@ def __init__(self, model_config=None, quantized=False): model_config quantized """ - parent_dir = "/".join(os.path.realpath(__file__).split("/")[:-1]) + self.parent_dir = str(pathlib.Path(os.path.abspath(__file__)).parent) self.cfg = defaultdict(lambda: None) if model_config: - config_filepath = parent_dir + "/model_cards/" + model_config + ".json" + config_filepath = os.path.join( + self.parent_dir, "model_cards", model_config + ".json" + ) with open(config_filepath) as f_in: self.cfg = json.load(f_in) Downloader.__init__( self, tar_url_post_opt_weights=self.cfg["artifacts"]["tar_url_post_opt_weights"], url_aimet_config=self.cfg["artifacts"]["url_aimet_config"], - model_dir=parent_dir, + model_dir=self.parent_dir, ) self.model = None self.quantized = quantized + if self.quantized: + self.model_name_or_path = os.path.join( + self.parent_dir, self.cfg["model_args"]["quantized"]["model_name_or_path"] + ) + else: + self.model_name_or_path = self.cfg["model_args"]["original"]["model_name_or_path"] + + self.config_file = os.path.join( + self.parent_dir, self.cfg["model_args"]["config_file"] + ) + self.dataset_name = os.path.join( + self.parent_dir, self.cfg["model_args"]["dataset_name"] + ) - def get_model_from_pretrained(self, dataset): + def get_model_from_pretrained(self,dataset): """get original or optmized model - Parameters: - dataset: Return: model : pretrained/optmized model """ @@ -68,24 +82,15 @@ def get_model_from_pretrained(self, dataset): label2id = {label: str(i) for i, label in enumerate(labels)} id2label = {str(i): label for i, label in enumerate(labels)} - if self.quantized: - model_name_or_path = self.cfg["model_args"]["quantized"][ - "model_name_or_path" - ] - else: - model_name_or_path = self.cfg["model_args"]["original"][ - "model_name_or_path" - ] - config = Config.from_pretrained( - model_name_or_path, + self.model_name_or_path, label2id=label2id, id2label=id2label, finetuning_task="image-classification", ) config.return_dict = False feature_extractor = FeatureExtractor.from_pretrained( - model_name_or_path, + self.model_name_or_path, ) # pylint:disable = unused-variable interpolate = False @@ -101,8 +106,8 @@ def get_model_from_pretrained(self, dataset): ) self.model = VitModel.from_pretrained( - model_name_or_path, - from_tf=bool(".ckpt" in model_name_or_path), + self.model_name_or_path, + from_tf=bool(".ckpt" in self.model_name_or_path), config=config, ignore_mismatched_sizes=ignore_mismatched_sizes, ) @@ -174,7 +179,7 @@ def get_quantsim(self, dataloader, eval_function): "quantization_configuration" ]["param_bw"], in_place=True, - config_file=self.cfg["model_args"]["config_file"], + config_file=self.config_file, ) quant_sim.compute_encodings(eval_function, [10, dataloader, metric]) diff --git a/packaging/aimet_zoo_tensorflow_pyproject.toml b/packaging/aimet_zoo_tensorflow_pyproject.toml index 6a69784..338890d 100644 --- a/packaging/aimet_zoo_tensorflow_pyproject.toml +++ b/packaging/aimet_zoo_tensorflow_pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "aimet_zoo_tensorflow" -version = "1.3.0" +version = "1.4.0" description = "Collection of popular TensorFlow neural network models and evaluators to quantize floating-point models using the AI Model Efficiency ToolKit (AIMET)." readme = "README.md" requires-python = ">= 3.6" diff --git a/packaging/aimet_zoo_torch_pyproject.toml b/packaging/aimet_zoo_torch_pyproject.toml index 8d6461f..62e2e94 100644 --- a/packaging/aimet_zoo_torch_pyproject.toml +++ b/packaging/aimet_zoo_torch_pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "aimet_zoo_torch" -version = "1.3.0" +version = "1.4.0" description = "Collection of popular PyTorch neural network models and evaluators to quantize floating-point models using the AI Model Efficiency ToolKit (AIMET)." readme = "README.md" requires-python = ">= 3.6" diff --git a/packaging/version.txt b/packaging/version.txt index f0bb29e..88c5fb8 100644 --- a/packaging/version.txt +++ b/packaging/version.txt @@ -1 +1 @@ -1.3.0 +1.4.0