align tflie and qnnpack

model_compression_toolkit/target_platform_capabilities/schema/mct_current_schema.py

@@ -7,6 +7,6 @@
 QuantizationConfigOptions = schema.QuantizationConfigOptions
 OperatorsSetBase = schema.OperatorsSetBase
 OperatorsSet = schema.OperatorsSet
-OperatorSetConcat= schema.OperatorSetConcat
+OperatorSetConcat = schema.OperatorSetConcat
 Fusing = schema.Fusing
 TargetPlatformModel = schema.TargetPlatformModel

model_compression_toolkit/target_platform_capabilities/schema/v1.py

@@ -66,6 +66,7 @@ class OperatorSetNames(Enum):
     OPSET_DROPOUT = "Dropout"
     OPSET_SPLIT_CHUNK = "SplitChunk"
     OPSET_MAXPOOL = "MaxPool"
+    OPSET_AVGPOOL = "AvgPool"
     OPSET_SIZE = "Size"
     OPSET_SHAPE = "Shape"
     OPSET_EQUAL = "Equal"
@@ -76,6 +77,9 @@ class OperatorSetNames(Enum):
     OPSET_CROPPING2D = "Cropping2D"
     OPSET_ZERO_PADDING2d = "ZeroPadding2D"
     OPSET_CAST = "Cast"
+    OPSET_RESIZE = "Resize"
+    OPSET_PAD = "Pad"
+    OPSET_FOLD = "Fold"
     OPSET_STRIDED_SLICE = "StridedSlice"
     OPSET_SSD_POST_PROCESS = "SSDPostProcess"
 

model_compression_toolkit/target_platform_capabilities/target_platform/targetplatform2framework/attach2keras.py

@@ -23,11 +23,11 @@
 
 if version.parse(tf.__version__) >= version.parse("2.13"):
     from keras.src.layers import Conv2D, DepthwiseConv2D, Dense, Reshape, ZeroPadding2D, Dropout, \
-        MaxPooling2D, Activation, ReLU, Add, Subtract, Multiply, PReLU, Flatten, Cropping2D, LeakyReLU, Permute, \
-        Conv2DTranspose, Identity, Concatenate, BatchNormalization, Minimum, Maximum, Softmax
+        MaxPooling2D, AveragePooling2D, Activation, ReLU, Add, Subtract, Multiply, PReLU, Flatten, Cropping2D, LeakyReLU, Permute, \
+        Conv2DTranspose, Concatenate, BatchNormalization, Minimum, Maximum, Softmax
 else:
     from keras.layers import Conv2D, DepthwiseConv2D, Dense, Reshape, ZeroPadding2D, Dropout, \
-        MaxPooling2D, Activation, ReLU, Add, Subtract, Multiply, PReLU, Flatten, Cropping2D, LeakyReLU, Permute, \
+        MaxPooling2D, AveragePooling2D, Activation, ReLU, Add, Subtract, Multiply, PReLU, Flatten, Cropping2D, LeakyReLU, Permute, \
         Conv2DTranspose, Concatenate, BatchNormalization, Minimum, Maximum, Softmax
 
 from model_compression_toolkit import DefaultDict
@@ -81,8 +81,12 @@ def __init__(self):
             OperatorSetNames.OPSET_SQUEEZE.value: [tf.squeeze],
             OperatorSetNames.OPSET_DROPOUT.value: [Dropout],
             OperatorSetNames.OPSET_SPLIT_CHUNK.value: [tf.split],
-            OperatorSetNames.OPSET_MAXPOOL.value: [MaxPooling2D],
+            OperatorSetNames.OPSET_MAXPOOL.value: [MaxPooling2D, tf.nn.avg_pool2d],
+            OperatorSetNames.OPSET_AVGPOOL.value: [AveragePooling2D],
             OperatorSetNames.OPSET_SIZE.value: [tf.size],
+            OperatorSetNames.OPSET_RESIZE.value: [tf.image.resize],
+            OperatorSetNames.OPSET_PAD.value: [tf.pad],
+            OperatorSetNames.OPSET_FOLD.value: [tf.space_to_batch_nd],
             OperatorSetNames.OPSET_SHAPE.value: [tf.shape, tf.compat.v1.shape],
             OperatorSetNames.OPSET_EQUAL.value: [tf.math.equal],
             OperatorSetNames.OPSET_ARGMAX.value: [tf.math.argmax],

model_compression_toolkit/target_platform_capabilities/target_platform/targetplatform2framework/attach2pytorch.py

@@ -20,8 +20,8 @@
     chunk, unbind, topk, gather, equal, transpose, permute, argmax, squeeze, multiply, subtract, minimum, \
     maximum, softmax
 from torch.nn import Conv2d, Linear, ConvTranspose2d, MaxPool2d, BatchNorm2d, Dropout, Flatten, Hardtanh, ReLU, ReLU6, \
-    PReLU, SiLU, Sigmoid, Tanh, Hardswish, Hardsigmoid, LeakyReLU, GELU, LogSoftmax, Softmax, ELU
-from torch.nn.functional import relu, relu6, prelu, silu, hardtanh, hardswish, hardsigmoid, leaky_relu, gelu
+    PReLU, SiLU, Sigmoid, Tanh, Hardswish, Hardsigmoid, LeakyReLU, GELU, LogSoftmax, Softmax, ELU, AvgPool2d
+from torch.nn.functional import relu, relu6, prelu, silu, hardtanh, hardswish, hardsigmoid, leaky_relu, gelu, fold
 import torch.nn.functional as F
 
 from model_compression_toolkit import DefaultDict
@@ -31,7 +31,6 @@
 from model_compression_toolkit.target_platform_capabilities.target_platform import LayerFilterParams, Eq
 from model_compression_toolkit.target_platform_capabilities.target_platform.targetplatform2framework.attach2fw import \
     AttachTpcToFramework
-from model_compression_toolkit.verify_packages import FOUND_SONY_CUSTOM_LAYERS
 
 
 class AttachTpcToPytorch(AttachTpcToFramework):
@@ -76,7 +75,11 @@ def __init__(self):
             OperatorSetNames.OPSET_DROPOUT.value: [Dropout, dropout],
             OperatorSetNames.OPSET_SPLIT_CHUNK.value: [split, chunk],
             OperatorSetNames.OPSET_MAXPOOL.value: [MaxPool2d, F.max_pool2d],
+            OperatorSetNames.OPSET_AVGPOOL.value: [AvgPool2d, F.avg_pool2d],
             OperatorSetNames.OPSET_SIZE.value: [torch.Tensor.size],
+            OperatorSetNames.OPSET_RESIZE.value: [torch.Tensor.resize],
+            OperatorSetNames.OPSET_PAD.value: [F.pad],
+            OperatorSetNames.OPSET_FOLD.value: [fold, F.fold],
             OperatorSetNames.OPSET_SHAPE.value: [torch.Tensor.shape],
             OperatorSetNames.OPSET_EQUAL.value: [equal],
             OperatorSetNames.OPSET_ARGMAX.value: [argmax],

model_compression_toolkit/target_platform_capabilities/tpc_models/qnnpack_tpc/v1/tp_model.py

@@ -155,12 +155,6 @@ def generate_tp_model(default_config: OpQuantizationConfig,
     relu = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_RELU.value)
     relu6 = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_RELU.value)
 
-    # TODO: missing this operator from relu in attach2fw which is part of the old Keras TPC.
-    #                                            tp.LayerFilterParams(ReLU, negative_slope=0.0),
-    #  need to figure out how to allow to retrive out built-int fw TPCs with a custom layer mapping
-    #  (preferably without changing the existing API for any user who used them).
-    #  consider adding a test for qnnpack and tflite tpcs
-
     hard_tanh = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_HARD_TANH.value)
     linear = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_FULLY_CONNECTED.value)
 

model_compression_toolkit/target_platform_capabilities/tpc_models/tflite_tpc/latest/__init__.py

@@ -15,7 +15,7 @@
 from model_compression_toolkit.verify_packages import FOUND_TORCH, FOUND_TF
 from model_compression_toolkit.target_platform_capabilities.tpc_models.tflite_tpc.v1.tp_model import get_tp_model, generate_tp_model, get_op_quantization_configs
 if FOUND_TF:
-    from model_compression_toolkit.target_platform_capabilities.tpc_models.tflite_tpc.v1.tpc_keras import get_keras_tpc as get_keras_tpc_latest
+    from model_compression_toolkit.target_platform_capabilities.tpc_models.tflite_tpc.v1.tp_model import get_keras_tpc as get_keras_tpc_latest
     from model_compression_toolkit.target_platform_capabilities.tpc_models.get_target_platform_capabilities import \
         get_tpc_model as generate_keras_tpc
 if FOUND_TORCH:

model_compression_toolkit/target_platform_capabilities/tpc_models/tflite_tpc/v1/tp_model.py

@@ -144,76 +144,76 @@ def generate_tp_model(default_config: OpQuantizationConfig,
     operator_set = []
     fusing_patterns = []
 
-    operator_set.append(schema.OperatorsSet("NoQuantization",
-                           default_configuration_options.clone_and_edit(
-                               quantization_preserving=True)))
-
-    fc = schema.OperatorsSet(schema.OperatorSetNames.OPSET_FULLY_CONNECTED.value,
-                             default_configuration_options.clone_and_edit_weight_attribute(
-                                 weights_per_channel_threshold=False))
-    sigmoid = schema.OperatorsSet(schema.OperatorSetNames.OPSET_SIGMOID.value,
-                             default_configuration_options.clone_and_edit_weight_attribute(
+    quant_preserving = default_configuration_options.clone_and_edit(quantization_preserving=True)
+
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_UNSTACK.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_TRANSPOSE.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_GATHER.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_RESHAPE.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_MAXPOOL.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_AVGPOOL.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_STRIDED_SLICE.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_CONCATENATE.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_MUL.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_MIN.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_MAX.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_ZERO_PADDING2d.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_RESIZE.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_PAD.value, qc_options=quant_preserving))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_FOLD.value, qc_options=quant_preserving))
+
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_L2NORM,
+                                            qc_options=default_configuration_options.clone_and_edit(
+                                                fixed_zero_point=0, fixed_scale=1 / 128)))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_LOG_SOFTMAX,
+                                            qc_options=default_configuration_options.clone_and_edit(
+                                                fixed_zero_point=127, fixed_scale=16 / 256)))
+    operator_set.append(schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_SOFTMAX,
+                                            qc_options=default_configuration_options.clone_and_edit(
+                                                fixed_zero_point=-128, fixed_scale=1 / 256)))
+
+    sigmoid = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_SIGMOID.value,
+                                  qc_options=default_configuration_options.clone_and_edit_weight_attribute(
+                                      weights_per_channel_threshold=False))
+    tanh = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_TANH.value,
+                               qc_options=default_configuration_options.clone_and_edit(
+                                   fixed_zero_point=-128, fixed_scale=1 / 256))
+    fc = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_FULLY_CONNECTED.value,
+                             qc_options=default_configuration_options.clone_and_edit_weight_attribute(
                                  weights_per_channel_threshold=False))
-    tanh = schema.OperatorsSet(schema.OperatorSetNames.OPSET_TANH.value, default_configuration_options.clone_and_edit(
-                               fixed_zero_point=-128, fixed_scale=1 / 256))
-    squeeze = schema.OperatorsSet("Squeeze",
+    squeeze = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_SQUEEZE,
                                   qc_options=default_configuration_options.clone_and_edit(
                                       quantization_preserving=True))
 
-    conv2d = schema.OperatorsSet(schema.OperatorSetNames.OPSET_CONV.value)
-    relu = schema.OperatorsSet(schema.OperatorSetNames.OPSET_RELU.value)
-    relu6 = schema.OperatorsSet(schema.OperatorSetNames.OPSET_RELU6.value)
-    batch_norm = schema.OperatorsSet(schema.OperatorSetNames.OPSET_BATCH_NORM.value)
-    add = schema.OperatorsSet(schema.OperatorSetNames.OPSET_ADD.value)
-
-    # TODO: in the old TPC the following operator sets were defined in the tflite TPC but it requires custom opset
-    #  implementation in the attach2fw side
-    #   FW:
-    #   tp.OperationsSetToLayers("L2Normalization", [tp.LayerFilterParams(torch.nn.functional.normalize, Eq('p', 2) | Eq('p', None))])
-    #   tp.OperationsSetToLayers("LogSoftmax", [torch.nn.LogSoftmax])
-    #   tp.OperationsSetToLayers("Elu", [torch.nn.ELU, torch.nn.functional.elu])
-    #   tp.OperationsSetToLayers("Softmax", [torch.nn.Softmax, torch.nn.functional.softmax])
-    #   HardTanh:
-    #       tp.LayerFilterParams(torch.nn.Hardtanh, min_val=0, max_val=6),
-    #       tp.LayerFilterParams(torch.nn.functional.hardtanh, min_val=0, max_val=6)
-    #   CFG:
-    #   operator_set.append(schema.OperatorsSet("L2Normalization",
-    #                            default_configuration_options.clone_and_edit(
-    #                                fixed_zero_point=0, fixed_scale=1 / 128)))
-    #   operator_set.append(schema.OperatorsSet("LogSoftmax",
-    #                            default_configuration_options.clone_and_edit(
-    #                                fixed_zero_point=127, fixed_scale=16 / 256)))
-    #   operator_set.append(schema.OperatorsSet("Softmax",
-    #                            default_configuration_options.clone_and_edit(
-    #                                fixed_zero_point=-128, fixed_scale=1 / 256)))
-    #   elu = schema.OperatorsSet("Elu")
-    #   hardtanh - same as relu and part of relu fuses
-    #   .
-    #   activations_to_fuse = schema.OperatorSetConcat([relu, elu])
-    #   operator_set.extend([fc, conv2d, relu, elu, batch_norm, bias_add, add, squeeze])
-
-    activations_to_fuse = schema.OperatorSetConcat([relu, relu6])
-    kernel = schema.OperatorSetConcat([conv2d, fc])
-
-    operator_set.extend([fc, conv2d, relu, relu6, tanh, sigmoid, batch_norm, add, squeeze])
+    conv2d = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_CONV.value)
+    relu = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_RELU.value)
+    relu6 = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_RELU6.value)
+    elu = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_ELU.value)
+    batch_norm = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_BATCH_NORM.value)
+    add = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_ADD.value)
+    bias_add = schema.OperatorsSet(name=schema.OperatorSetNames.OPSET_ADD_BIAS.value)
+
+    kernel = schema.OperatorSetConcat(operators_set=[conv2d, fc])
+    activations_to_fuse = schema.OperatorSetConcat(operators_set=[relu, elu])
+
+    operator_set.extend([fc, conv2d, relu, relu6, tanh, sigmoid, batch_norm, add, bias_add, elu, squeeze])
 
     # ------------------- #
     # Fusions
     # ------------------- #
     # Source: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/grappler/optimizers/remapper
-    # TODO: bias_add was removed since there is no connection to bias-add operator on the fw side
-    # fusing_patterns.append(schema.Fusing((kernel, bias_add)))
-    # fusing_patterns.append(schema.Fusing((kernel, bias_add, activations_to_fuse)))
-    fusing_patterns.append(schema.Fusing((conv2d, batch_norm, activations_to_fuse)))
-    fusing_patterns.append(schema.Fusing((conv2d, squeeze, activations_to_fuse)))
-    fusing_patterns.append(schema.Fusing((batch_norm, activations_to_fuse)))
-    fusing_patterns.append(schema.Fusing((batch_norm, add, activations_to_fuse)))
+    fusing_patterns.append(schema.Fusing(operator_groups=(kernel, bias_add)))
+    fusing_patterns.append(schema.Fusing(operator_groups=(kernel, bias_add, activations_to_fuse)))
+    fusing_patterns.append(schema.Fusing(operator_groups=(conv2d, batch_norm, activations_to_fuse)))
+    fusing_patterns.append(schema.Fusing(operator_groups=(conv2d, squeeze, activations_to_fuse)))
+    fusing_patterns.append(schema.Fusing(operator_groups=(batch_norm, activations_to_fuse)))
+    fusing_patterns.append(schema.Fusing(operator_groups=(batch_norm, add, activations_to_fuse)))
 
     # Create a TargetPlatformModel and set its default quantization config.
     # This default configuration will be used for all operations
     # unless specified otherwise (see OperatorsSet, for example):
     generated_tpc = schema.TargetPlatformModel(
-        default_configuration_options,
+        default_qco=default_configuration_options,
         tpc_minor_version=1,
         tpc_patch_version=0,
         operator_set=tuple(operator_set),