Merge pull request #96 from christophbrgr/master

Fixes label list return error

.gitignore

@@ -8,3 +8,4 @@
 .coverage
 .vscode
 *.code-workspace
+/framework/htmlcov/

framework/modelhubapi_tests/mockmodels/contrib_src_mi/inference.py

@@ -0,0 +1,43 @@
+"""
+Implementation of several mock models to test the API. Each model has a
+slightly different behaviour, which should be properly handled by the API.
+Also most models are not fully valid, e.g. they do not comply to the mock
+config. This is ok for unit testing, most models are only used for a small
+set of specific tests requiring that model's specific behavioural aspect.
+
+These models test absed on the multi input mock model (contrib_src_mi) with
+a single output.
+"""
+
+import os
+import numpy as np
+from modelhublib.model import ModelBase
+
+
+class Model(ModelBase):
+
+    def __init__(self):
+        pass
+
+    def infer(self, input):
+        pass
+
+class ModelReturnsOneLabelList(ModelBase):
+
+    def __init__(self):
+        pass
+
+    def infer(self, input):
+        if os.path.isfile(input):
+            # Return a constant classification result no matter what's the input
+            label_list = [{"label": "class_0", "probability": 0.3},
+                          {"label": "class_1", "probability": 0.7}]
+            return label_list
+        else:
+            raise IOError("File " + input + " does not exist.")
+
+
+class ModelReturnsListOfOneLabelList(ModelReturnsOneLabelList):
+
+    def infer(self, input):
+        return [super(ModelReturnsListOfOneLabelList, self).infer(input)]

framework/modelhubapi_tests/mockmodels/contrib_src_mi/model/config.json

@@ -0,0 +1,57 @@
+{
+  "id": "MockId",
+  "meta": {
+    "name": "MockNet",
+    "application_area": "UnitTest",
+    "task": "Mock for Unit Testing",
+    "task_extended":
+      "Simple mock model for unit testing the APIs",
+    "data_type": "Image",
+    "data_source": "None"
+  },
+  "publication": {
+    "title":
+      "Mock Title",
+    "journal": "None",
+    "year": 1111,
+    "authors":
+      "No One",
+    "email": "mock@mock.org",
+    "abstract":
+      "Mock abstract",
+    "url": "https://modelhub.ai",
+    "google_scholar":
+      "",
+    "bibtex":
+      ""
+  },
+  "model": {
+    "description":
+      "Mock description",
+    "architecture": "Empty",
+    "learning_type": "None",
+    "io": {
+      "input": {
+        "format": ["image/png"],
+        "dim_limits": [
+          {
+            "min": 1,
+            "max": 4
+          },
+          {
+            "min": 1
+          },
+          {
+            "min": 1
+          }
+        ]
+      },
+      "output": [
+        {
+          "name": "probabilities",
+          "type": "label_list"
+        }
+      ]
+    }
+  }
+}

framework/modelhubapi_tests/mockmodels/contrib_src_mi/model/model.txt

@@ -0,0 +1,3 @@
+EMPTY MOCK MODEL FOR UNIT TESTING
+
+This model has only one return format

framework/modelhubapi_tests/mockmodel/contrib_src/inference.py → framework/modelhubapi_tests/mockmodels/contrib_src_si/inference.py

@@ -1,9 +1,12 @@
 """
-Implementation of several mock models to test the API. Each model has a 
+Implementation of several mock models to test the API. Each model has a
 slightly different behaviour, which should be properly handled by the API.
 Also most models are not fully valid, e.g. they do not comply to the mock
 config. This is ok for unit testing, most models are only used for a small
 set of specific tests requiring that model's specific behavioural aspect.
+
+These models test based on the single input mock model (contrib_src_si) with
+multiple outputs.
 """
 
 import os
@@ -14,7 +17,7 @@
 class Model(ModelBase):
 
     def __init__(self):
-        pass    
+        pass
 
     def infer(self, input):
         if os.path.isfile(input):
@@ -30,7 +33,7 @@ def infer(self, input):
 class ModelReturnsOneNumpyArray(ModelBase):
 
     def __init__(self):
-        pass    
+        pass
 
     def infer(self, input):
         if os.path.isfile(input):
@@ -50,13 +53,13 @@ def infer(self, input):
 class ModelReturnsOneLabelList(ModelBase):
 
     def __init__(self):
-        pass    
+        pass
 
     def infer(self, input):
         if os.path.isfile(input):
             # Return a constant classification result no matter what's the input
-            label_list = [{"label": "class_0", 'probability': 0.3},
-                          {"label": "class_1", 'probability': 0.7}]
+            label_list = [{"label": "class_0", "probability": 0.3},
+                          {"label": "class_1", "probability": 0.7}]
             return label_list
         else:
             raise IOError("File " + input + " does not exist.")
@@ -71,8 +74,7 @@ def infer(self, input):
 class ModelThrowingError(ModelBase):
 
     def __init__(self):
-        pass    
+        pass
 
     def infer(self, input):
         raise NotImplementedError
-

framework/modelhubapi_tests/pythonapi_test.py

@@ -4,18 +4,18 @@
 import shutil
 from modelhubapi import ModelHubAPI
 from .apitestbase import TestAPIBase
-from .mockmodel.contrib_src.inference import Model
-from .mockmodel.contrib_src.inference import ModelReturnsOneNumpyArray, ModelReturnsOneLabelList
-from .mockmodel.contrib_src.inference import ModelReturnsListOfOneNumpyArray, ModelReturnsListOfOneLabelList
-
+from .mockmodels.contrib_src_si.inference import Model
+from .mockmodels.contrib_src_si.inference import ModelReturnsOneNumpyArray
+from .mockmodels.contrib_src_si.inference import ModelReturnsListOfOneNumpyArray, ModelReturnsListOfOneLabelList
+from .mockmodels.contrib_src_mi.inference import ModelReturnsOneLabelList
 
 
 class TestModelHubAPI(TestAPIBase):
 
     def setUp(self):
         model = Model()
         self.this_dir = os.path.dirname(os.path.realpath(__file__))
-        contrib_src_dir = os.path.join(self.this_dir, "mockmodel", "contrib_src")
+        contrib_src_dir = os.path.join(self.this_dir, "mockmodels", "contrib_src_si")
         self.api = ModelHubAPI(model, contrib_src_dir)
         self.setup_self_temp_output_dir()
         self.api.output_folder = self.temp_output_dir
@@ -53,28 +53,28 @@ def test_get_model_io_returns_expected_mock_values(self):
 
     def test_get_samples_returns_path_to_mock_samples(self):
         samples = self.api.get_samples()
-        self.assertEqual(self.this_dir + "/mockmodel/contrib_src/sample_data", samples["folder"])
+        self.assertEqual(self.this_dir + "/mockmodels/contrib_src_si/sample_data", samples["folder"])
         samples["files"].sort()
         self.assertListEqual(["testimage_ramp_4x2.jpg",
                               "testimage_ramp_4x2.png"],
                              samples["files"])
 
 
     def test_predict_returns_expected_mock_prediction_list(self):
-        result = self.api.predict(self.this_dir + "/mockmodel/contrib_src/sample_data/testimage_ramp_4x2.png", numpyToFile=False)
+        result = self.api.predict(self.this_dir + "/mockmodels/contrib_src_si/sample_data/testimage_ramp_4x2.png", numpyToFile=False)
         self.assert_predict_contains_expected_mock_prediction(result, expectList=True)
 
     def test_predict_returns_expected_mock_prediction_url(self):
-        result = self.api.predict(self.this_dir + "/mockmodel/contrib_src/sample_data/testimage_ramp_4x2.png", numpyToFile=True)
+        result = self.api.predict(self.this_dir + "/mockmodels/contrib_src_si/sample_data/testimage_ramp_4x2.png", numpyToFile=True)
         self.assert_predict_contains_expected_mock_prediction(result)
 
     def test_predict_returns_expected_mock_meta_info(self):
-        result = self.api.predict(self.this_dir + "/mockmodel/contrib_src/sample_data/testimage_ramp_4x2.png")
+        result = self.api.predict(self.this_dir + "/mockmodels/contrib_src_si/sample_data/testimage_ramp_4x2.png")
         self.assert_predict_contains_expected_mock_meta_info(result)
 
 
     def test_predict_returns_correct_output_format(self):
-        result = self.api.predict(self.this_dir + "/mockmodel/contrib_src/sample_data/testimage_ramp_4x2.png", numpyToFile=False)
+        result = self.api.predict(self.this_dir + "/mockmodels/contrib_src_si/sample_data/testimage_ramp_4x2.png", numpyToFile=False)
         self.assertIsInstance(result["output"], list)
         self.assertIsInstance(result["output"][0]["prediction"], list)
         self.assertIsInstance(result["output"][0]["prediction"][0], dict)
@@ -84,7 +84,7 @@ def test_predict_returns_correct_output_format(self):
 
     def test_predict_output_types_match_config(self):
         model_io = self.api.get_model_io()
-        result = self.api.predict(self.this_dir + "/mockmodel/contrib_src/sample_data/testimage_ramp_4x2.png")
+        result = self.api.predict(self.this_dir + "/mockmodels/contrib_src_si/sample_data/testimage_ramp_4x2.png")
         self.assertEqual(len(model_io["output"]), len(result["output"]))
         for i in range(len(model_io["output"])):
             self.assertEqual(model_io["output"][i]["type"], result["output"][i]["type"])
@@ -96,7 +96,7 @@ class TestModelHubAPIModelReturnsOneNumpyArray(unittest.TestCase):
     def setUp(self):
         model = ModelReturnsOneNumpyArray()
         self.this_dir = os.path.dirname(os.path.realpath(__file__))
-        contrib_src_dir = os.path.join(self.this_dir, "mockmodel", "contrib_src")
+        contrib_src_dir = os.path.join(self.this_dir, "mockmodels", "contrib_src_si")
         self.api = ModelHubAPI(model, contrib_src_dir)
 
 
@@ -105,12 +105,12 @@ def tearDown(self):
 
 
     def test_predict_returns_expected_mock_prediction_list(self):
-        result = self.api.predict(self.this_dir + "/mockmodel/contrib_src/sample_data/testimage_ramp_4x2.png", numpyToFile=False)
+        result = self.api.predict(self.this_dir + "/mockmodels/contrib_src_si/sample_data/testimage_ramp_4x2.png", numpyToFile=False)
         self.assertListEqual([[0,1,1,0],[0,2,2,0]], result["output"][0]["prediction"])
 
 
     def test_predict_returns_correct_output_format(self):
-        result = self.api.predict(self.this_dir + "/mockmodel/contrib_src/sample_data/testimage_ramp_4x2.png", numpyToFile=False)
+        result = self.api.predict(self.this_dir + "/mockmodels/contrib_src_si/sample_data/testimage_ramp_4x2.png", numpyToFile=False)
         self.assertIsInstance(result["output"], list)
         self.assertIsInstance(result["output"][0]["prediction"], list)
 
@@ -121,7 +121,7 @@ class TestModelHubAPIModelReturnsListOfOneNumpyArray(TestModelHubAPIModelReturns
     def setUp(self):
         model = ModelReturnsListOfOneNumpyArray()
         self.this_dir = os.path.dirname(os.path.realpath(__file__))
-        contrib_src_dir = os.path.join(self.this_dir, "mockmodel", "contrib_src")
+        contrib_src_dir = os.path.join(self.this_dir, "mockmodels", "contrib_src_si")
         self.api = ModelHubAPI(model, contrib_src_dir)
 
 
@@ -131,24 +131,26 @@ class TestModelHubAPIModelReturnsOneLabelList(unittest.TestCase):
     def setUp(self):
         model = ModelReturnsOneLabelList()
         self.this_dir = os.path.dirname(os.path.realpath(__file__))
-        contrib_src_dir = os.path.join(self.this_dir, "mockmodel", "contrib_src")
+        # load config version 2 with only one output specified
+        contrib_src_dir = os.path.join(self.this_dir, "mockmodels", "contrib_src_mi")
         self.api = ModelHubAPI(model, contrib_src_dir)
 
 
+
     def tearDown(self):
         pass
 
 
     def test_predict_returns_expected_mock_prediction(self):
-        result = self.api.predict(self.this_dir + "/mockmodel/contrib_src/sample_data/testimage_ramp_4x2.png")
-        self.assertEqual("class_0", result["output"][0]["prediction"][0]["label"])
+        result = self.api.predict(self.this_dir + "/mockmodels/contrib_src_si/sample_data/testimage_ramp_4x2.png")
         self.assertEqual(0.3, result["output"][0]["prediction"][0]["probability"])
         self.assertEqual("class_1", result["output"][0]["prediction"][1]["label"])
         self.assertEqual(0.7, result["output"][0]["prediction"][1]["probability"])
+        self.assertEqual("class_0", result["output"][0]["prediction"][0]["label"])
 
 
     def test_predict_returns_correct_output_format(self):
-        result = self.api.predict(self.this_dir + "/mockmodel/contrib_src/sample_data/testimage_ramp_4x2.png")
+        result = self.api.predict(self.this_dir + "/mockmodels/contrib_src_si/sample_data/testimage_ramp_4x2.png")
         self.assertIsInstance(result["output"], list)
         self.assertIsInstance(result["output"][0]["prediction"], list)
         self.assertIsInstance(result["output"][0]["prediction"][0], dict)
@@ -161,7 +163,7 @@ class TestModelHubAPIModelReturnsListOfOneLabelList(TestModelHubAPIModelReturnsO
     def setUp(self):
         model = ModelReturnsListOfOneLabelList()
         self.this_dir = os.path.dirname(os.path.realpath(__file__))
-        contrib_src_dir = os.path.join(self.this_dir, "mockmodel", "contrib_src")
+        contrib_src_dir = os.path.join(self.this_dir, "mockmodels", "contrib_src_si")
         self.api = ModelHubAPI(model, contrib_src_dir)
 
 

framework/modelhubapi_tests/pythonapivoid_test.py

@@ -3,15 +3,15 @@
 import unittest
 import os
 from modelhubapi import ModelHubAPI
-from .mockmodel.contrib_src.inference import ModelThrowingError
+from .mockmodels.contrib_src_si.inference import ModelThrowingError
 
 
 class TestModelHubAPIVoidModel(unittest.TestCase):
 
     def setUp(self):
         model = ModelThrowingError()
         self.this_dir = os.path.dirname(os.path.realpath(__file__))
-        void_contrib_src_dir = os.path.join(self.this_dir, "mockmodel", "void_contrib_src")
+        void_contrib_src_dir = os.path.join(self.this_dir, "mockmodels", "void_contrib_src")
         self.api = ModelHubAPI(model, void_contrib_src_dir)
 
 

framework/modelhubapi_tests/restapi_test.py

@@ -3,7 +3,7 @@
 from zipfile import ZipFile
 import shutil
 import json
-from modelhubapi_tests.mockmodel.contrib_src.inference import Model
+from modelhubapi_tests.mockmodels.contrib_src_si.inference import Model
 from .apitestbase import TestRESTAPIBase
 
 
@@ -12,7 +12,7 @@ class TestModelHubRESTAPI(TestRESTAPIBase):
 
     def setUp(self):
         self.this_dir = os.path.dirname(os.path.realpath(__file__))
-        self.contrib_src_dir = os.path.join(self.this_dir, "mockmodel", "contrib_src")
+        self.contrib_src_dir = os.path.join(self.this_dir, "mockmodels", "contrib_src_si")
         self.setup_self_temp_work_dir()
         self.setup_self_temp_output_dir()
         self.setup_self_test_client(Model(), self.contrib_src_dir)

framework/modelhubapi_tests/restapivoid_test.py

@@ -6,7 +6,7 @@
 import shutil
 import json
 import six
-from modelhubapi_tests.mockmodel.contrib_src.inference import ModelThrowingError
+from modelhubapi_tests.mockmodels.contrib_src_si.inference import ModelThrowingError
 from .apitestbase import TestRESTAPIBase
 
 
@@ -63,7 +63,7 @@ class TestModelHubRESTAPIModelThrowingError(TestRESTAPIBase):
 
     def setUp(self):
         self.this_dir = os.path.dirname(os.path.realpath(__file__))
-        self.contrib_src_dir = os.path.join(self.this_dir, "mockmodel", "contrib_src")
+        self.contrib_src_dir = os.path.join(self.this_dir, "mockmodels", "contrib_src_si")
         self.setup_self_temp_work_dir()
         self.setup_self_temp_output_dir()
         self.setup_self_test_client(ModelThrowingError(), self.contrib_src_dir)

framework/modelhublib_tests/imageconverters_tests/numpyToNumpyConverter_test.py

@@ -0,0 +1,27 @@
+import unittest
+import os
+import json
+import PIL
+import numpy as np
+
+from modelhublib.imageconverters import NumpyToNumpyConverter
+
+class TestNumpyImageConverter(unittest.TestCase):
+
+    def setUp(self):
+        self.imageConverter = NumpyToNumpyConverter()
+
+    def tearDown(self):
+        pass
+
+    def test_convert_success_on_arr(self):
+        baseline = np.asarray([[1,2,3,4],[2,0,0,0],[1,2,3,4]])
+        npArr = self.imageConverter.convert(baseline)
+        self.assertEqual(baseline.shape, npArr.shape)
+
+    def test_convert_fails_on_image_as_input(self):
+        image = PIL.Image.new("RGB", (64,32))
+        self.assertRaises(IOError, self.imageConverter.convert, image)
+
+if __name__ == '__main__':
+    unittest.main()