TVM Support for Accuracy Checker

models/public/resnet-50-pytorch/accuracy-check.yml

@@ -62,3 +62,36 @@ models:
           # Image channels must be swapped, because "pillow_imread" reads in RGB, but converted model expect BGR
           - type: rgb_to_bgr
           # Reference metric from PyTorch (pytorch v1.0.1, torchvision v0.2.2) top-1 76.13% top-5 92.862%
+
+  - name: resnet-50-pytorch
+    launchers:
+      - framework: tvm
+        model: public/resnet-50-pytorch/resnet-v1-50.dylib
+        adapter: classification
+        inputs:
+          - name: data
+            type: INPUT
+            shape: 1,3,224,224
+        _input_precision:
+            - data:FP32
+
+    datasets:
+      - name: imagenet_1000_classes
+        reader: pillow_imread
+        preprocessing:
+          - type: resize
+            size: 256
+            aspect_ratio_scale: greater
+            use_pillow: true
+            interpolation: BILINEAR
+
+          - type: crop
+            size: 224
+            use_pillow: true
+
+          - type: normalization
+            std: 255
+
+          - type: normalization
+            mean: (0.485, 0.456, 0.406)
+            std: (0.229, 0.224, 0.225)

models/public/resnet-50-tf/accuracy-check.yml

@@ -34,3 +34,30 @@ models:
             aspect_ratio_scale: greater
           - type: crop
             size: 224
+
+  - name: resnet-50-tvm
+    launchers:
+      - framework: tvm
+        model: public/resnet-50-tf/resnet_v1-50.dylib
+        adapter: classification
+        inputs:
+          - name: map/TensorArrayStack/TensorArrayGatherV3
+            type: INPUT
+            shape: 1,224,224,3
+            layout: NHWC
+        _input_precision:
+          - map/TensorArrayStack/TensorArrayGatherV3:FP32
+        batch: 1
+        dev_id: 0
+
+    datasets:
+      - name: imagenet_1001_classes
+        preprocessing:
+          - type: bgr_to_rgb
+          - type: normalization
+            mean: [123.68, 116.78, 103.94]
+          - type: resize
+            size: 256
+            aspect_ratio_scale: greater
+          - type: crop
+            size: 224

tools/accuracy_checker/accuracy_checker/launcher/__init__.py

@@ -53,6 +53,13 @@
         'tf_lite', "TensorFlow isn't installed. Please, install it before using. \n{}".format(import_error.msg)
     )
 
+try:
+    from .tvm_launcher import TVMLauncher
+except ImportError as import_error:
+    TVMLauncher = unsupported_launcher(
+        'tvm', "TVM isn't installed. Please, install it before using. \n{}".format(import_error.msg)
+    )
+
 try:
     from .tf2_launcher import TF2Launcher
 except ImportError as import_error:
@@ -78,6 +85,7 @@
     'MxNetLauncher',
     'TFLauncher',
     'TFLiteLauncher',
+    'TVMLauncher',
     'DLSDKLauncher',
     'OpenCVLauncher',
     'ONNXLauncher',

tools/accuracy_checker/accuracy_checker/launcher/tvm_launcher.py

@@ -0,0 +1,86 @@
+import os
+from collections import OrderedDict
+
+from ..config import NumberField
+from .launcher import Launcher
+
+
+class TVMLauncher(Launcher):
+    __provider__ = 'tvm'
+
+    @classmethod
+    def parameters(cls):
+        parameters = super().parameters()
+        parameters.update({
+            'batch': NumberField(value_type=int, min_value=1, optional=True, description="Batch size.", default=1),
+            'dev_id': NumberField(value_type=int, min_value=0, optional=True, description="Device ID.", default=0),
+        })
+        return parameters
+
+    def __init__(self, config_entry: dict, *args, **kwargs):
+        super().__init__(config_entry, *args, **kwargs)
+        try:
+            import tvm  # pylint: disable=C0415
+            import tvm.rpc  # pylint: disable=C0415
+            import tvm.contrib.graph_executor  # pylint: disable=C0415
+            self._tvm = tvm
+            self._tvm_rpc = tvm.rpc
+            self._tvm_runtime = tvm.contrib.graph_executor
+        except ImportError as import_error:
+            raise ValueError("TVM isn't installed. Please, install it before using. \n{}".format(import_error.msg))
+        self.validate_config(config_entry)
+
+        self._remote_session = self._tvm_rpc.LocalSession()
+        self._device = self._remote_session.cpu(self.get_value_from_config('dev_id'))
+        self._module = self._load_module(config_entry['model'])
+
+        self._batch = self.get_value_from_config('batch')
+        self._generate_inputs()
+        self._outputs_names = list(range(self._module.get_num_outputs()))
+
+    def _generate_inputs(self):
+        config_inputs = self.config.get('inputs')
+        input_shapes = OrderedDict()
+        for input_description in config_inputs:
+            input_shapes[input_description['name']] = input_description.get('shape', (self.batch,) + (-1,) * 3)
+        self._inputs = input_shapes
+
+    def _load_module(self, model_path):
+        model_name = os.path.split(model_path)[-1]
+        self._remote_session.upload(model_path)
+        lib = self._remote_session.load_module(model_name)
+        return self._tvm_runtime.GraphModule(lib["default"](self._device))
+
+    @property
+    def inputs(self):
+        return self._inputs
+
+    @property
+    def batch(self):
+        return self._batch
+
+    @property
+    def output_blob(self):
+        return next(iter(self._outputs_names))
+
+    def fit_to_input(self, data, layer_name, layout, precision):
+        data = super().fit_to_input(data, layer_name, layout, precision)
+        return self._tvm.nd.array(data, self._device)
+
+    def predict(self, inputs, metadata=None, **kwargs):
+        results = []
+        for batch_input in inputs:
+            for input_name, input_data in batch_input.items():
+                self._module.set_input(input_name, input_data)
+            self._module.run()
+            results.append({output_name: self._module.get_output(output_name).asnumpy()
+                            for output_name in self._outputs_names})
+        return results
+
+    def predict_async(self, *args, **kwargs):
+        raise ValueError('TVM Launcher does not support async mode yet')
+
+    def release(self):
+        del self._module
+        del self._device
+        del self._remote_session

tools/downloader/mo_tvm.py

@@ -0,0 +1,128 @@
+import argparse
+import sys
+from functools import partial
+import fnmatch
+
+import tvm
+from tvm import relay
+from tvm.driver.tvmc.frontends import load_model
+
+import common
+from accuracy_checker.utils import get_path, cast_to_bool, check_file_existence, validate_print_interval, string_to_list
+
+
+def add_common_args(parser):
+    common_args = parser.add_argument_group('Common arguments')
+    common_args.add_argument('--framework',
+                             help='Name of the framework used to train the input model.',
+                             type=str,
+                             choices=['tf', 'caffe', 'mxnet', 'kaldi', 'onnx'])
+    common_args.add_argument('--data_type', type=str)
+    common_args.add_argument('--output_dir', type=str)
+    common_args.add_argument('--model_name', type=str)
+    common_args.add_argument('--input', type=str)
+    common_args.add_argument('--output', type=str)
+    common_args.add_argument('--input_model', type=str)
+    common_args.add_argument('--target', type=str)
+    common_args.add_argument('--mean_values', '-ms',
+                              help='Mean values to be used for the input image per channel. ' +
+                                   'Values to be provided in the (R,G,B) or [R,G,B] format. ' +
+                                   'Can be defined for desired input of the model, for example: ' +
+                                   '"--mean_values data[255,255,255],info[255,255,255]". ' +
+                                   'The exact meaning and order ' +
+                                   'of channels depend on how the original model was trained.',
+                              default=())
+    common_args.add_argument('--scale_values',
+                              help='Scale values to be used for the input image per channel. ' +
+                                   'Values are provided in the (R,G,B) or [R,G,B] format. ' +
+                                   'Can be defined for desired input of the model, for example: ' +
+                                   '"--scale_values data[255,255,255],info[255,255,255]". ' +
+                                   'The exact meaning and order ' +
+                                   'of channels depend on how the original model was trained.',
+                              default=())
+    common_args.add_argument('--reverse_input_channels',
+                              help='Switch the input channels order from RGB to BGR (or vice versa). Applied to '
+                                   'original inputs of the model if and only if a number of channels equals 3. Applied '
+                                   'after application of --mean_values and --scale_values options, so numbers in '
+                                   '--mean_values and --scale_values go in the order of channels used in the original '
+                                   'model.',
+                              action='store_true')
+
+
+def expand_arguments(parser):
+    argv = parser.parse_args()
+    model_info = get_model_info(argv)
+    shape = get_shape_from_model_info(model_info)
+
+    expanded_group = parser.add_argument_group('Expanded arguments')
+    expanded_group.add_argument('--shape_dict', type=dict, default={argv.input: shape}, required=False)
+
+
+def build_arguments_parser():
+    parser = argparse.ArgumentParser(description='Deep Learning accuracy validation framework', allow_abbrev=False)
+    add_common_args(parser)
+    expand_arguments(parser)
+
+    return parser
+
+
+def get_model_info(args):
+    return [model for model in common.load_models(args)
+            if fnmatch.fnmatchcase(model.name, args.model_name)][0]
+
+
+def get_shape_from_string(string):
+    processed = string.replace(' ', '')
+    processed = processed.replace('[', '')
+    processed = processed.replace(']', '')
+    processed = processed.split(',')
+    return list(int(entry) for entry in processed)
+
+
+def get_shape_from_model_info(model_info):
+    if model_info.conversion_to_onnx_args:
+        args_with_shape = model_info.conversion_to_onnx_args
+        shape_pattern = '--input-shape='
+    else:
+        args_with_shape = model_info.mo_args
+        shape_pattern = '--input_shape='
+
+    str_shape = [arg for arg in args_with_shape if shape_pattern in arg][0]
+    str_shape = str_shape.replace(shape_pattern, '')
+    return get_shape_from_string(str_shape)
+
+
+def main(cli_parser: argparse.ArgumentParser):
+    argv = cli_parser.parse_args()
+    print(argv)
+
+    model = load_model(argv.input_model, argv.framework, argv.shape_dict)
+    print(model.mod)
+    model.export_classic_format()
+
+    target = 'llvm'
+    target_host = target
+    arch = "x86_64"
+    sdk = "macosx"
+    from tvm.contrib import xcode
+
+    with tvm.transform.PassContext(opt_level=3, config={"relay.backend.use_auto_scheduler": True}):
+        lib = relay.build(model.mod, target=target, target_host=target_host, params=model.params)
+        lib.export_library("/Users/agladyshev/workspace/open_model_zoo/public/resnet-50-pytorch/resnet-v1-50.dylib", fcompile=xcode.create_dylib, arch=arch, sdk=sdk)
+
+    from tvm.contrib import graph_executor
+    import numpy as np
+    #
+    dtype = "float32"
+    dev = tvm.cpu(0)
+    m = graph_executor.GraphModule(lib["default"](dev))
+    m.set_input(argv.input, tvm.nd.array(np.zeros((1, 224, 224, 3), dtype=dtype)))
+    m.run()
+    tvm_output = m.get_output(0).asnumpy()
+    print(tvm_output.shape)
+
+
+if __name__ == '__main__':
+    # from mo.utils.cli_parser import get_all_cli_parser
+    # sys.exit(main(get_all_cli_parser()))
+    sys.exit(main(build_arguments_parser()))