ModelManager, Model and ModelVersion classes

config.json

@@ -0,0 +1,25 @@
+{
+    "models": [{
+        "name": "Resnet",
+        "path": "/models/resnet.xml",
+        "version": 1,
+        "backend": "CPU",
+        "batchSize": 1,
+        "shape": [1, 3, 224, 224]
+    },{
+        "name": "Resnet",
+        "path": "/models/resnet.xml",
+        "version": 2,
+        "backend": "CPU",
+        "batchSize": 1,
+        "shape": [1, 3, 224, 224]
+    },
+    {
+        "name": "OthernetModel",
+        "path": "/models/resnet.xml",
+        "version": 1,
+        "backend": "CPU",
+        "batchSize": 1,
+        "shape": [1, 3, 224, 224]
+    }]
+}

src/BUILD

@@ -21,11 +21,19 @@ cc_binary(
     name = "server_cc",
     srcs = [
         "server.cc",
+        "model.cpp",
+        "model.h",
+        "modelmanager.cpp",
+        "modelmanager.h",
+        "modelversion.cpp",
+        "modelversion.h",
+         "status.h",
     ],
     deps = [
         "//tensorflow_serving/apis:prediction_service_cc_proto",
         "@com_github_grpc_grpc//:grpc++",
         "@org_tensorflow//tensorflow/core:framework",
+        "@com_github_tencent_rapidjson//:rapidjson",
         "@openvino//:headers",
         "@openvino//:library",
     ],

src/model.cpp

@@ -0,0 +1,41 @@
+//*****************************************************************************
+// Copyright 2020 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+#include "model.h"
+
+namespace ovms {
+
+Status Model::addVersion(   const std::string& name,
+                            const std::string& path,
+                            const std::string& backend,
+                            const int64_t version,
+                            const size_t batchSize,
+                            const std::vector<size_t>& shape) {
+    std::shared_ptr<ModelVersion> modelVersion = std::make_shared<ModelVersion>();
+    auto status = modelVersion->loadModel(path, backend, version, batchSize, shape);
+    if (status != Status::OK) {
+        return status;
+    }
+    this->name = name;
+    modelVersions.push_back(std::move(modelVersion));
+
+    return Status::OK;
+}
+
+Status Model::dropVersion(const ModelVersion& modelVersion) {
+    return Status::OK;
+}
+
+} // namespace ovms

src/model.h

@@ -0,0 +1,106 @@
+//*****************************************************************************
+// Copyright 2020 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "modelversion.h"
+
+namespace ovms {
+
+    /**
+     * @brief This class represent inference models
+     */
+    class Model {
+        protected:
+            /**
+             * @brief Model name
+             */
+            std::string name;
+
+            /**
+             * @brief Default version of the model
+             */
+            unsigned int defaultVersion;
+
+            /**
+             * @brief Model input
+             */
+            std::vector<std::shared_ptr<ModelVersion>> modelVersions;
+
+        public:
+            /**
+             * @brief A default constructor
+             */
+            Model() = default;
+
+            /**
+             * @brief Gets the model name
+             * 
+             * @return model name
+             */
+            const std::string& getName() {
+                return name;
+            }
+
+            /**
+             * @brief Gets model default version
+             * 
+             * @return default model version
+             */
+            unsigned int getDefaultVersion() {
+                return defaultVersion;
+            }
+
+            /**
+             * @brief Gets model versions
+             *
+             * @return model versions
+             */
+            const std::vector<std::shared_ptr<ModelVersion>>& getModelVersions() const {
+                return modelVersions;
+            }
+
+            /**
+             * @brief Adds a new version of ModelVersion to the list of versions
+             * 
+             * @param name model name
+             * @param path to the model
+             * @param backend
+             * @param version 
+             * @param batchSize
+             * @param shape
+             *  
+             * @return status
+             */
+            Status addVersion(  const std::string& name,
+                                const std::string& path,
+                                const std::string& backend,
+                                const int64_t version,
+                                const size_t batchSize,
+                                const std::vector<size_t>& shape);
+
+            /**
+             * @brief Removes model version from the list
+             * 
+             * @param modelVersion object
+             * 
+             * @return status
+             */
+            Status dropVersion(const ModelVersion& modelVersion);
+    };
+} // namespace ovms

src/modelmanager.cpp

@@ -0,0 +1,82 @@
+//*****************************************************************************
+// Copyright 2020 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+#include <fstream>
+
+#include <rapidjson/document.h>
+#include <rapidjson/istreamwrapper.h>
+
+#include "modelmanager.h"
+
+namespace ovms {
+
+Status ModelManager::start(const std::string& jsonFilename) {
+    rapidjson::Document doc;
+    std::ifstream ifs(jsonFilename.c_str());
+
+    // Perform some basic checks on the config file
+    if (!ifs.good()) {
+        // Logger(Log::Error, "File is invalid ", jsonFilename);
+        return Status::FILE_INVALID;
+    }
+
+    rapidjson::IStreamWrapper isw(ifs);
+    if (doc.ParseStream(isw).HasParseError()) {
+        // Logger(Log::Error, "Configuration file is not a valid JSON file.");
+        return Status::JSON_INVALID;
+    }
+
+    // TODO validate json against schema
+    const auto itr = doc.FindMember("models");
+    if (itr == doc.MemberEnd() || !itr->value.IsArray()) {
+        // Logger(Log::Error, "Configuration file doesn't have models property.");
+        return Status::JSON_INVALID;
+    }
+
+    for (auto& v : itr->value.GetArray()) {
+        Model model;
+        std::string name = v["name"].GetString();
+        if (models.find(name) == models.end()) {
+            models[name] = model;
+        }
+
+        std::vector<size_t> shape;
+        for (auto& s : v["shape"].GetArray()) {
+            shape.push_back(s.GetUint64());
+        }
+
+        auto status = models[name].addVersion(v["name"].GetString(),
+                                              v["path"].GetString(),
+                                              v["backend"].GetString(),
+                                              v["version"].GetInt64(),
+                                              v["batchSize"].GetUint64(),
+                                              shape);
+        if (status != Status::OK) {
+            // Logger(Log::Warning, "There was an error loading a model ", v["name"].GetString());
+        }
+    }
+
+    return Status::OK;
+}
+
+Status ModelManager::join() {
+    if (monitor.joinable()) {
+        monitor.join();
+    }
+
+    return Status::OK;
+}
+
+} // namespace ovms

src/modelmanager.h

@@ -0,0 +1,96 @@
+//*****************************************************************************
+// Copyright 2020 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+#pragma once
+
+#include <string>
+#include <thread>
+
+#include "model.h"
+
+namespace ovms {
+    /**
+     * @brief Model manager is managing the list of model topologies enabled for serving and their versions.
+     */
+    class ModelManager {
+        private:
+            /**
+             * @brief A default constructor is private
+             */
+            ModelManager() = default;
+
+            /**
+             * @brief Private copying constructor
+             */
+            ModelManager(const ModelManager&);
+
+            /**
+             * @brief A JSON configuration filename
+             */
+            std::string configFilename;
+
+            /**
+             * @brief A collection of models
+             */
+            std::map<std::string, Model> models;
+
+            /**
+             * @brief A thread object used for monitoring changes in config
+             */
+            std::thread monitor;
+        public:
+            /**
+             * @brief Gets the instance of ModelManager
+             */
+            static ModelManager& getInstance() {
+                static ModelManager instance;
+
+                return instance;
+            }
+
+            /**
+             * @brief Gets config filename
+             * 
+             * @return config filename
+             */
+            const std::string& getConfigFilename() {
+                return configFilename;
+            }
+
+            /**
+             * @brief Gets models collection
+             * 
+             * @return models collection
+             */
+            const std::map<std::string, Model>& getModels() {
+                return models;
+            }
+
+            /**
+             * @brief Starts model manager using provided config
+             * 
+             * @param filename
+             * @return status
+             */
+            Status start(const std::string& jsonFilename);
+
+            /**
+             * @brief Gracefully finish the thread
+             * 
+             * @return status
+             */
+            Status join();
+    };
+}  // namespace ovms