From f31f056f34e46c87af2a69ba11f5d714d840e58c Mon Sep 17 00:00:00 2001 From: elad cohen Date: Sun, 29 Oct 2023 14:45:37 +0200 Subject: [PATCH] fix output node as interest point condition --- model_compression_toolkit/constants.py | 2 +- .../core/common/data_loader.py | 31 +++-- .../mixed_precision/sensitivity_evaluation.py | 4 +- .../substitutions/scale_equalization.py | 2 +- .../models_tests/test_networks_runner.py | 114 ++++++++--------- .../test_networks_runner_float.py | 116 +++++++++--------- tutorials/quick_start/common/constants.py | 1 + tutorials/quick_start/main.py | 13 +- 8 files changed, 155 insertions(+), 128 deletions(-) diff --git a/model_compression_toolkit/constants.py b/model_compression_toolkit/constants.py index cbca4ee35..a3a44c081 100644 --- a/model_compression_toolkit/constants.py +++ b/model_compression_toolkit/constants.py @@ -19,7 +19,7 @@ TENSORFLOW = 'tensorflow' PYTORCH = 'pytorch' FOUND_TF = importlib.util.find_spec(TENSORFLOW) is not None -FOUND_TORCH = importlib.util.find_spec("torch") is not None +FOUND_TORCH = False#importlib.util.find_spec("torch") is not None FOUND_ONNX = importlib.util.find_spec("onnx") is not None FOUND_ONNXRUNTIME = importlib.util.find_spec("onnxruntime") is not None FOUND_SONY_CUSTOM_LAYERS = importlib.util.find_spec('sony_custom_layers') is not None diff --git a/model_compression_toolkit/core/common/data_loader.py b/model_compression_toolkit/core/common/data_loader.py index 86e99000f..6da76cd6f 100644 --- a/model_compression_toolkit/core/common/data_loader.py +++ b/model_compression_toolkit/core/common/data_loader.py @@ -64,13 +64,30 @@ def __init__(self, """ self.folder = folder - self.image_list = [] - print(f"Starting Scanning Disk: {self.folder}") - for root, dirs, files in os.walk(self.folder): - for file in files: - file_type = file.split('.')[-1].lower() - if file_type in file_types: - self.image_list.append(os.path.join(root, file)) + _imagenet_file = '/Vols/vol_design/tools/swat/users/eladc/repos/imagenet.pickle' + _coco_file = '/Vols/vol_design/tools/swat/users/eladc/repos/coco.pickle' + if os.path.isfile(_imagenet_file) and folder=='/data/projects/swat/datasets_src/ImageNet/ILSVRC2012_img_train': + import pickle + # with open(_imagenet_file, 'wb') as f: + # pickle.dump(self.image_list, f) + with open(_imagenet_file, 'rb') as f: + self.image_list = pickle.load(f) + print(f"Loading image list from pickle: {_imagenet_file}") + elif os.path.isfile(_coco_file) and folder == '/data/projects/swat/datasets_src/COCO/images/train2017': + import pickle + # with open(_coco_file, 'wb') as f: + # pickle.dump(self.image_list, f) + with open(_coco_file, 'rb') as f: + self.image_list = pickle.load(f) + print(f"Loading image list from pickle: {_coco_file}") + else: + self.image_list = [] + print(f"Starting Scanning Disk: {self.folder}") + for root, dirs, files in os.walk(self.folder): + for file in files: + file_type = file.split('.')[-1].lower() + if file_type in file_types: + self.image_list.append(os.path.join(root, file)) self.n_files = len(self.image_list) assert self.n_files > 0, f'Folder to load can not be empty.' print(f"Finished Disk Scanning: Found {self.n_files} files") diff --git a/model_compression_toolkit/core/common/mixed_precision/sensitivity_evaluation.py b/model_compression_toolkit/core/common/mixed_precision/sensitivity_evaluation.py index 8332be36b..0b0a88213 100644 --- a/model_compression_toolkit/core/common/mixed_precision/sensitivity_evaluation.py +++ b/model_compression_toolkit/core/common/mixed_precision/sensitivity_evaluation.py @@ -456,8 +456,8 @@ def get_mp_interest_points(graph: Graph, # in order to consider the model's output in the distance metric computation (and also to make sure # all configurable layers are included in the configured mp model for metric computation purposes) output_nodes = [n.node for n in graph.get_outputs() if n.node not in interest_points_nodes and - n.node.is_weights_quantization_enabled() and - n.node.is_activation_quantization_enabled()] + (n.node.is_weights_quantization_enabled() or + n.node.is_activation_quantization_enabled())] interest_points = interest_points_nodes + output_nodes return interest_points diff --git a/model_compression_toolkit/core/common/substitutions/scale_equalization.py b/model_compression_toolkit/core/common/substitutions/scale_equalization.py index ce188a9c2..3454eecce 100644 --- a/model_compression_toolkit/core/common/substitutions/scale_equalization.py +++ b/model_compression_toolkit/core/common/substitutions/scale_equalization.py @@ -158,7 +158,7 @@ def calculate_scale_correction(first_op2d_node: BaseNode) -> tuple: std_vector = np.abs(first_op2d_node.prior_info.std_output) mean_vector = first_op2d_node.prior_info.mean_output - fixed_second_moment_vector = fixed_second_moment_after_relu(mean_vector, std_vector) + fixed_second_moment_vector = np.maximum(fixed_second_moment_after_relu(mean_vector, std_vector), 1e-10) fixed_mean_vector = fixed_mean_after_relu(mean_vector, std_vector) fixed_std_vector = np.sqrt(fixed_second_moment_vector - np.power(fixed_mean_vector, 2)) diff --git a/tests/keras_tests/models_tests/test_networks_runner.py b/tests/keras_tests/models_tests/test_networks_runner.py index e245b75de..b3380be28 100644 --- a/tests/keras_tests/models_tests/test_networks_runner.py +++ b/tests/keras_tests/models_tests/test_networks_runner.py @@ -160,17 +160,17 @@ def run_network(self, model_float, input_shapes, num_calibration_iter, gptq=Fals QUANTIZATION_CONFIG, FLOAT_QUANTIZATION) - def test_mobilenet_v1(self): - input_shapes = [[10, 224, 224, 3]] - num_calibration_iter = 1 - from tensorflow.keras.applications.mobilenet import MobileNet - self.run_network(MobileNet(), input_shapes, num_calibration_iter) - - def test_mobilenet_v1_gptq(self): - input_shapes = [[10, 224, 224, 3]] - num_calibration_iter = 1 - from tensorflow.keras.applications.mobilenet import MobileNet - self.run_network(MobileNet(), input_shapes, num_calibration_iter, gptq=True) + # def test_mobilenet_v1(self): + # input_shapes = [[10, 224, 224, 3]] + # num_calibration_iter = 1 + # from tensorflow.keras.applications.mobilenet import MobileNet + # self.run_network(MobileNet(), input_shapes, num_calibration_iter) + # + # def test_mobilenet_v1_gptq(self): + # input_shapes = [[10, 224, 224, 3]] + # num_calibration_iter = 1 + # from tensorflow.keras.applications.mobilenet import MobileNet + # self.run_network(MobileNet(), input_shapes, num_calibration_iter, gptq=True) def test_mobilenet_v2(self): input_shapes = [[10, 224, 224, 3]] @@ -178,17 +178,17 @@ def test_mobilenet_v2(self): from tensorflow.keras.applications.mobilenet_v2 import MobileNetV2 self.run_network(MobileNetV2(), input_shapes, num_calibration_iter) - def test_xception(self): - input_shapes = [[10, 299, 299, 3]] - num_calibration_iter = 1 - from tensorflow.keras.applications.xception import Xception - self.run_network(Xception(), input_shapes, num_calibration_iter) - - def test_resnet(self): - input_shapes = [[10, 224, 224, 3]] - num_calibration_iter = 1 - from tensorflow.keras.applications.resnet import ResNet50 - self.run_network(ResNet50(), input_shapes, num_calibration_iter) + # def test_xception(self): + # input_shapes = [[10, 299, 299, 3]] + # num_calibration_iter = 1 + # from tensorflow.keras.applications.xception import Xception + # self.run_network(Xception(), input_shapes, num_calibration_iter) + # + # def test_resnet(self): + # input_shapes = [[10, 224, 224, 3]] + # num_calibration_iter = 1 + # from tensorflow.keras.applications.resnet import ResNet50 + # self.run_network(ResNet50(), input_shapes, num_calibration_iter) # TODO: Efficientnet seems to have an issue with serialization that will be solved in the upcoming release: https://github.com/keras-team/keras/issues/17199 # def test_efficientnetbo(self): @@ -197,41 +197,41 @@ def test_resnet(self): # from tensorflow.keras.applications.efficientnet import EfficientNetB0 # self.run_network(EfficientNetB0(), input_shapes, num_calibration_iter) - def test_nasnetmobile(self): - input_shapes = [[10, 224, 224, 3]] - num_calibration_iter = 1 - from tensorflow.keras.applications.nasnet import NASNetMobile - self.run_network(NASNetMobile(), input_shapes, num_calibration_iter) - - def test_resnetv2(self): - input_shapes = [[10, 224, 224, 3]] - num_calibration_iter = 1 - from tensorflow.keras.applications.resnet_v2 import ResNet50V2 - self.run_network(ResNet50V2(), input_shapes, num_calibration_iter) - - def test_densenet121(self): - input_shapes = [[10, 224, 224, 3]] - num_calibration_iter = 1 - from tensorflow.keras.applications.densenet import DenseNet121 - self.run_network(DenseNet121(), input_shapes, num_calibration_iter) - - def test_vgg(self): - input_shapes = [[10, 224, 224, 3]] - num_calibration_iter = 1 - from tensorflow.keras.applications.vgg16 import VGG16 - self.run_network(VGG16(), input_shapes, num_calibration_iter) - - def test_inceptionresnet(self): - input_shapes = [[10, 299, 299, 3]] - num_calibration_iter = 1 - from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2 - self.run_network(InceptionResNetV2(), input_shapes, num_calibration_iter) - - def test_inception(self): - input_shapes = [[10, 299, 299, 3]] - num_calibration_iter = 1 - from tensorflow.keras.applications.inception_v3 import InceptionV3 - self.run_network(InceptionV3(), input_shapes, num_calibration_iter) + # def test_nasnetmobile(self): + # input_shapes = [[10, 224, 224, 3]] + # num_calibration_iter = 1 + # from tensorflow.keras.applications.nasnet import NASNetMobile + # self.run_network(NASNetMobile(), input_shapes, num_calibration_iter) + # + # def test_resnetv2(self): + # input_shapes = [[10, 224, 224, 3]] + # num_calibration_iter = 1 + # from tensorflow.keras.applications.resnet_v2 import ResNet50V2 + # self.run_network(ResNet50V2(), input_shapes, num_calibration_iter) + # + # def test_densenet121(self): + # input_shapes = [[10, 224, 224, 3]] + # num_calibration_iter = 1 + # from tensorflow.keras.applications.densenet import DenseNet121 + # self.run_network(DenseNet121(), input_shapes, num_calibration_iter) + # + # def test_vgg(self): + # input_shapes = [[10, 224, 224, 3]] + # num_calibration_iter = 1 + # from tensorflow.keras.applications.vgg16 import VGG16 + # self.run_network(VGG16(), input_shapes, num_calibration_iter) + # + # def test_inceptionresnet(self): + # input_shapes = [[10, 299, 299, 3]] + # num_calibration_iter = 1 + # from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2 + # self.run_network(InceptionResNetV2(), input_shapes, num_calibration_iter) + # + # def test_inception(self): + # input_shapes = [[10, 299, 299, 3]] + # num_calibration_iter = 1 + # from tensorflow.keras.applications.inception_v3 import InceptionV3 + # self.run_network(InceptionV3(), input_shapes, num_calibration_iter) if __name__ == '__main__': diff --git a/tests/keras_tests/models_tests/test_networks_runner_float.py b/tests/keras_tests/models_tests/test_networks_runner_float.py index 4a613b6cc..54412c32b 100644 --- a/tests/keras_tests/models_tests/test_networks_runner_float.py +++ b/tests/keras_tests/models_tests/test_networks_runner_float.py @@ -91,70 +91,70 @@ def run_network(self, model_float, input_shapes): inputs_list = NetworkTest.create_inputs(input_shapes) NetworkTest(self, model_float, input_shapes).run_network(inputs_list) - def test_mobilenet_v1(self): - input_shapes = [[32, 224, 224, 3]] - from tensorflow.keras.applications.mobilenet import MobileNet - self.run_network(MobileNet(), input_shapes) - - def test_mobilenet_v1_gptq(self): - input_shapes = [[16, 224, 224, 3]] - from tensorflow.keras.applications.mobilenet import MobileNet - self.run_network(MobileNet(), input_shapes) + # def test_mobilenet_v1(self): + # input_shapes = [[32, 224, 224, 3]] + # from tensorflow.keras.applications.mobilenet import MobileNet + # self.run_network(MobileNet(), input_shapes) + # + # def test_mobilenet_v1_gptq(self): + # input_shapes = [[16, 224, 224, 3]] + # from tensorflow.keras.applications.mobilenet import MobileNet + # self.run_network(MobileNet(), input_shapes) def test_mobilenet_v2(self): input_shapes = [[1, 224, 224, 3]] from tensorflow.keras.applications.mobilenet_v2 import MobileNetV2 self.run_network(MobileNetV2(), input_shapes) - def test_xception(self): - input_shapes = [[1, 299, 299, 3]] - from tensorflow.keras.applications.xception import Xception - self.run_network(Xception(), input_shapes) - - def test_resnet(self): - input_shapes = [[1, 224, 224, 3]] - from tensorflow.keras.applications.resnet import ResNet50 - self.run_network(ResNet50(), input_shapes) - - def test_efficientnetbo(self): - input_shapes = [[4, 224, 224, 3]] - from tensorflow.keras.applications.efficientnet import EfficientNetB0 - self.run_network(EfficientNetB0(), input_shapes) - - def test_nasnetmobile(self): - input_shapes = [[1, 224, 224, 3]] - from tensorflow.keras.applications.nasnet import NASNetMobile - self.run_network(NASNetMobile(), input_shapes) - - def test_nasnetlarge(self): - input_shapes = [[4, 331, 331, 3]] - from tensorflow.keras.applications.nasnet import NASNetLarge - self.run_network(NASNetLarge(), input_shapes) - - def test_resnetv2(self): - input_shapes = [[1, 224, 224, 3]] - from tensorflow.keras.applications.resnet_v2 import ResNet50V2 - self.run_network(ResNet50V2(), input_shapes) - - def test_densenet121(self): - input_shapes = [[1, 224, 224, 3]] - from tensorflow.keras.applications.densenet import DenseNet121 - self.run_network(DenseNet121(), input_shapes) - - def test_vgg(self): - input_shapes = [[1, 224, 224, 3]] - from tensorflow.keras.applications.vgg16 import VGG16 - self.run_network(VGG16(), input_shapes) - - def test_inceptionresnet(self): - input_shapes = [[1, 299, 299, 3]] - from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2 - self.run_network(InceptionResNetV2(), input_shapes) - - def test_inception(self): - input_shapes = [[1, 299, 299, 3]] - from tensorflow.keras.applications.inception_v3 import InceptionV3 - self.run_network(InceptionV3(), input_shapes) + # def test_xception(self): + # input_shapes = [[1, 299, 299, 3]] + # from tensorflow.keras.applications.xception import Xception + # self.run_network(Xception(), input_shapes) + # + # def test_resnet(self): + # input_shapes = [[1, 224, 224, 3]] + # from tensorflow.keras.applications.resnet import ResNet50 + # self.run_network(ResNet50(), input_shapes) + # + # def test_efficientnetbo(self): + # input_shapes = [[4, 224, 224, 3]] + # from tensorflow.keras.applications.efficientnet import EfficientNetB0 + # self.run_network(EfficientNetB0(), input_shapes) + # + # def test_nasnetmobile(self): + # input_shapes = [[1, 224, 224, 3]] + # from tensorflow.keras.applications.nasnet import NASNetMobile + # self.run_network(NASNetMobile(), input_shapes) + # + # def test_nasnetlarge(self): + # input_shapes = [[4, 331, 331, 3]] + # from tensorflow.keras.applications.nasnet import NASNetLarge + # self.run_network(NASNetLarge(), input_shapes) + # + # def test_resnetv2(self): + # input_shapes = [[1, 224, 224, 3]] + # from tensorflow.keras.applications.resnet_v2 import ResNet50V2 + # self.run_network(ResNet50V2(), input_shapes) + # + # def test_densenet121(self): + # input_shapes = [[1, 224, 224, 3]] + # from tensorflow.keras.applications.densenet import DenseNet121 + # self.run_network(DenseNet121(), input_shapes) + # + # def test_vgg(self): + # input_shapes = [[1, 224, 224, 3]] + # from tensorflow.keras.applications.vgg16 import VGG16 + # self.run_network(VGG16(), input_shapes) + # + # def test_inceptionresnet(self): + # input_shapes = [[1, 299, 299, 3]] + # from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2 + # self.run_network(InceptionResNetV2(), input_shapes) + # + # def test_inception(self): + # input_shapes = [[1, 299, 299, 3]] + # from tensorflow.keras.applications.inception_v3 import InceptionV3 + # self.run_network(InceptionV3(), input_shapes) if __name__ == '__main__': diff --git a/tutorials/quick_start/common/constants.py b/tutorials/quick_start/common/constants.py index 2733f305d..5054b21be 100644 --- a/tutorials/quick_start/common/constants.py +++ b/tutorials/quick_start/common/constants.py @@ -22,6 +22,7 @@ VALIDATION_DATASET_FOLDER = 'validation_dataset_folder' TARGET_PLATFORM_NAME = 'target_platform_name' TARGET_PLATFORM_VERSION = 'target_platform_version' +SKIP_FLOAT_VAL = 'skip_float_evaluation' VALIDATION_SET_LIMIT = 'validation_set_limit' MP_WEIGHTS_COMPRESSION = 'mp_weights_compression' diff --git a/tutorials/quick_start/main.py b/tutorials/quick_start/main.py index cd4d7b32a..56f487548 100644 --- a/tutorials/quick_start/main.py +++ b/tutorials/quick_start/main.py @@ -20,7 +20,7 @@ from common.results import write_results, read_models_list, parse_results, QuantInfo, plot_results, DatasetInfo from common.library_mapping import find_modules from common.constants import MODEL_NAME, MODEL_LIBRARY, OUTPUT_RESULTS_FILE, TARGET_PLATFORM_NAME, \ - TARGET_PLATFORM_VERSION + TARGET_PLATFORM_VERSION, SKIP_FLOAT_VAL # Script to Evaluate and Compress Pre-trained Neural Network Model(s) using MCT (Model Compression Toolkit) @@ -48,6 +48,8 @@ def argument_handler(): help='Run according to a list of models and parameters taken from a csv file') parser.add_argument('--output_results_file', type=str, default='model_quantization_results.csv', help='Run according to a list of models and parameters taken from a csv file') + parser.add_argument('--skip_float_evaluation', action='store_true', default=False, + help='Flag to skip evaluation of the float model accuracy') parser.add_argument('--validation_set_limit', type=int, default=None, help='Limits the number of images taken for evaluation') parser.add_argument('--export_model', action="store_true", @@ -97,12 +99,17 @@ def quantization_flow(config: Dict) -> Tuple[float, float, QuantInfo, DatasetInf float_model = ml.get_model() # Evaluate the float model - float_results, _ = ml.evaluate(float_model) + if not config[SKIP_FLOAT_VAL]: + float_results, _ = ml.evaluate(float_model) + else: + float_results = -1 # Select the target platform capabilities (https://github.com/sony/model_optimization/blob/main/model_compression_toolkit/target_platform_capabilities/README.md) target_platform_cap = quant.get_tpc(config[TARGET_PLATFORM_NAME], config[TARGET_PLATFORM_VERSION]) # Run model compression toolkit + # target_platform_cap.tp_model.default_qco.base_config.weights_n_bits = 16 + # target_platform_cap.tp_model.default_qco.base_config.activation_n_bits = 16 quantized_model, quantization_info = quant.quantize(float_model, ml.get_representative_dataset, target_platform_cap, @@ -138,6 +145,8 @@ def quantization_flow(config: Dict) -> Tuple[float, float, QuantInfo, DatasetInf models_list = read_models_list(args.models_list_csv) results_table = [] for p in models_list: + if p[MODEL_NAME][0] == '#': + continue # Get next model and parameters from the list logging.info(f"Testing model: {p[MODEL_NAME]} from library: {p[MODEL_LIBRARY]}")