onnx -> h5

Author: Semyon1104

app/Converters/parser.py

@@ -1,69 +1,79 @@
 import json
 import os
-
 import tensorflow as tf
 from tensorflow.keras.initializers import GlorotUniform as OriginalGlorotUniform, Zeros as OriginalZeros
 from tensorflow.keras.models import load_model
 
+
 class CustomGlorotUniform(OriginalGlorotUniform):
     def __init__(self, seed=None, **kwargs):
-        kwargs.pop('dtype', None)  # Remove the unexpected dtype keyword if present
+        kwargs.pop('dtype', None)
         super().__init__(seed=seed, **kwargs)
 
+
 class CustomZeros(OriginalZeros):
     def __init__(self, **kwargs):
-        kwargs.pop('dtype', None)  # Remove the unexpected dtype keyword if present
+        kwargs.pop('dtype', None)
         super().__init__(**kwargs)
 
-# Пути к модели и JSON файлу
+
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 MODEL_PATH = os.path.join(BASE_DIR, 'docs\\models', 'AlexNet-model.h5')
 MODEL_DATA_PATH = os.path.join(BASE_DIR, 'docs\\jsons', 'model_data_alexnet_1.json')
 
-# Загрузка модели
 model = load_model(MODEL_PATH, custom_objects={'GlorotUniform': CustomGlorotUniform, 'Zeros': CustomZeros})
 
-# Получение весов модели и информации о порядке слоев
 layer_info = []
 for index, layer in enumerate(model.layers):
     layer_name = layer.name
-    layer_type = type(layer).__name__  # Тип слоя (например, Conv2D, Dense, Activation и т.д.)
+    layer_type = type(layer).__name__
     layer_config = layer.get_config()
+    weights = layer.get_weights()
 
-    # Извлечение параметров слоя
-    layer_padding = None
-    layer_activation = None
-
-    if isinstance(layer, tf.keras.layers.Conv2D):
-        layer_padding = layer_config.get('padding', None)  # Считываем padding у Conv2D
-        layer_activation = layer_config.get('activation', None)  # Получаем функцию активации
-
-    # Сохранение информации о слое: его тип, имя, padding и веса
     layer_data = {
-        'index': len(layer_info),  # Порядковый номер слоя
+        'index': len(layer_info),
         'name': layer_name,
-        'type': layer_type
+        'type': layer_type,
+        'weights': [],
+        'bias': []
     }
 
-    if layer_padding is not None:
-        layer_data['padding'] = layer_padding
+    # Обработка параметров слоя
+    if isinstance(layer, tf.keras.layers.Conv2D):
+        layer_data['padding'] = layer_config.get('padding', None)
+        layer_activation = layer_config.get('activation', None)
+
+        # Для Conv2D веса имеют форму (kernel_size[0], kernel_size[1], in_channels, out_channels)
+        if len(weights) > 0:
+            layer_data['weights'] = weights[0].tolist()
+        if len(weights) > 1:
+            layer_data['bias'] = weights[1].tolist()
+
+    elif isinstance(layer, tf.keras.layers.Dense):
+        if len(weights) > 0:
+            layer_data['weights'] = weights[0].tolist()
+        if len(weights) > 1:
+            layer_data['bias'] = weights[1].tolist()
 
-    layer_data['weights'] = [w.tolist() for w in layer.get_weights()]
+    else:
+        # Для других слоев сохраняем все веса как есть
+        if weights:
+            layer_data['weights'] = [w.tolist() for w in weights]
 
     layer_info.append(layer_data)
 
-    # Если активация встроена в слой, добавляем её как отдельный слой
-    if layer_activation and not isinstance(layer, tf.keras.layers.Activation):
+    # Обработка встроенной активации
+    if isinstance(layer, (tf.keras.layers.Conv2D)) and layer_activation:
         activation_layer = {
             'index': len(layer_info),
             'name': f"activation_{layer_name}",
             'type': layer_activation,
-            'weights': []
+            'weights': [],
+            'bias': []
         }
         layer_info.append(activation_layer)
 
-# Сохранение данных в JSON файл
 with open(MODEL_DATA_PATH, 'w') as f:
     json.dump(layer_info, f, indent=2)
 
-print(f"Model data saved to {MODEL_DATA_PATH}")
+print(f"Model data saved to {MODEL_DATA_PATH}")

app/Converters/reader_weights_sample.cpp

@@ -16,7 +16,7 @@ int main() {
 
     try {
       Tensor tensor =
-          create_tensor_from_json(layer_data["weights"], Type::kFloat);
+          create_tensor_from_json(layer_data, Type::kFloat);
       // std::cout << tensor << std::endl;
     } catch (const std::exception& e) {
       std::cerr << "Error processing layer " << layer_name << ": " << e.what()

app/Graph/build.cpp

@@ -36,7 +36,7 @@ void build_graph(Tensor& input, Tensor& output, bool comments,
       std::cout << "Processing layer of type: " << layer_type << std::endl;
 
     Tensor tensor =
-        create_tensor_from_json(layer_data["weights"], Type::kFloat);
+        create_tensor_from_json(layer_data, Type::kFloat);
 
     if (layer_type.find("Conv") != std::string::npos) {
       Tensor tmp_tensor = tensor;

include/Weights_Reader/reader_weights.hpp

@@ -8,8 +8,6 @@ using json = nlohmann::json;
 using namespace itlab_2023;
 
 json read_json(const std::string& filename);
-void extract_values_without_bias(const json& j, std::vector<float>& values);
 void extract_values_from_json(const json& j, std::vector<float>& values);
 void parse_json_shape(const json& j, std::vector<size_t>& shape, size_t dim);
 Tensor create_tensor_from_json(const json& j, Type type);
-void extract_bias_from_json(const json& j, std::vector<float>& bias);

include/Weights_Reader/reader_weights_onnx.hpp

@@ -12,6 +12,3 @@ void extract_values_from_json_onnx(const json& j, std::vector<float>& values);
 void parse_json_shape_onnx(const json& j, std::vector<size_t>& shape,
                            size_t dim);
 Tensor create_tensor_from_json_onnx(const json& j, Type type);
-
-void parse_onnx_weights(const json& j, std::vector<float>& weights,
-                        std::vector<float>& bias);

src/Weights_Reader/reader_weights.cpp

@@ -9,22 +9,14 @@
 using json = nlohmann::json;
 
 json read_json(const std::string& filename) {
-  std::ifstream ifs(filename, std::ifstream::binary);
+  std::ifstream ifs(filename);
   if (!ifs.is_open()) {
     throw std::runtime_error("Failed to open JSON file: " + filename);
   }
 
-  ifs.seekg(0, std::ios::end);
-  size_t size = ifs.tellg();
-  ifs.seekg(0, std::ios::beg);
-
-  std::vector<char> buffer(size);
-  ifs.read(buffer.data(), size);
-  ifs.close();
-
   json model_data;
   try {
-    model_data = json::parse(buffer.begin(), buffer.end());
+    ifs >> model_data;
   } catch (const json::parse_error& e) {
     throw std::runtime_error("JSON parse error: " + std::string(e.what()));
   }
@@ -42,74 +34,49 @@ void extract_values_from_json(const json& j, std::vector<float>& values) {
   }
 }
 
-void extract_values_without_bias(const json& j, std::vector<float>& values) {
-  std::vector<float> temp_values;
-  extract_values_from_json(j, temp_values);
-  size_t bias_size = 0;
-  if (j.is_array() && !j.empty() && j.back().is_array()) {
-    bias_size = j.back().size();
+void parse_json_shape(const json& j, std::vector<size_t>& shape,
+                           size_t dim = 0) {
+  if (!j.is_array()) {
+    if (dim == 0) shape.push_back(0);
+    return;
   }
-  // std::cout << "Bias size: " << bias_size << std::endl;
-  if (temp_values.size() >= bias_size) {
-    values.assign(temp_values.begin(), temp_values.end() - bias_size);
+
+  if (shape.size() <= dim) {
+    shape.push_back(j.size());
   }
-}
 
-void parse_json_shape(const json& j, std::vector<size_t>& shape, size_t dim) {
-  if (dim == 0) {
-    if (j.is_array()) {
-      if (j.empty()) {
-        shape.push_back(0);
-        return;
-      }
-      parse_json_shape(j.front(), shape, dim + 1);
-    } else {
-      shape.push_back(0);
-    }
-  } else {
-    if (j.is_array()) {
-      if (shape.size() <= dim - 1) {
-        shape.push_back(j.size());
-      }
-      if (!j.empty()) {
-        parse_json_shape(j.front(), shape, dim + 1);
-      }
-    }
+  if (!j.empty()) {
+    parse_json_shape(j[0], shape, dim + 1);
   }
 }
 
-void extract_bias_from_json(const json& j, std::vector<float>& bias) {
-  if (j.is_array()) {
-    if (!j.empty() && j.back().is_array()) {
-      for (const auto& item : j.back()) {
-        if (item.is_number()) {
-          bias.push_back(item.get<float>());
-        }
-      }
-    }
+Tensor create_tensor_from_json(const json& layer_data, Type type) {
+  if (type != Type::kFloat) {
+    throw std::invalid_argument("Only float type is supported");
   }
-}
 
-Tensor create_tensor_from_json(const json& j, Type type) {
-  if (type == Type::kFloat) {
-    std::vector<float> vals;
-    std::vector<size_t> shape;
-    std::vector<float> bias;
-    extract_values_without_bias(j, vals);
-    // std::cout << "Extracted values size: " << vals.size() << std::endl;
+  std::vector<float> weights;
+  if (layer_data.contains("weights") && !layer_data["weights"].empty()) {
+    extract_values_from_json(layer_data["weights"], weights);
+  }
 
-    parse_json_shape(j, shape, 0);
-    // std::cout << "Shape: ";
-    /*for (size_t i = 0; i < shape.size() - 1; i++) {
-      std::cout << shape[i] << ", ";
-    }
-    std::cout << shape[shape.size() - 1];
-    std::cout << std::endl;*/
+  // Извлекаем bias (если есть)
+  std::vector<float> bias;
+  if (layer_data.contains("bias") && !layer_data["bias"].empty()) {
+    extract_values_from_json(layer_data["bias"], bias);
+  }
 
-    extract_bias_from_json(j, bias);
-    // std::cout << "Extracted bias size: " << bias.size() << std::endl;
-    Shape sh(shape);
-    return make_tensor<float>(vals, sh, bias);
+  // Определяем shape
+  std::vector<size_t> shape;
+  if (layer_data.contains("weights")) {
+    parse_json_shape(layer_data["weights"], shape);
   }
-  throw std::invalid_argument("Unsupported type or invalid JSON format");
+
+  std::cout << "Extracted weights size: " << weights.size() << std::endl;
+  std::cout << "Shape: ";
+  for (auto dim : shape) std::cout << dim << " ";
+  std::cout << std::endl;
+  std::cout << "Extracted bias size: " << bias.size() << std::endl;
+
+  return make_tensor<float>(weights, Shape(shape), bias);
 }

src/Weights_Reader/reader_weights_onnx.cpp

@@ -72,11 +72,11 @@ Tensor create_tensor_from_json_onnx(const json& layer_data, Type type) {
     parse_json_shape_onnx(layer_data["weights"], shape);
   }
 
-  std::cout << "Extracted weights size: " << weights.size() << std::endl;
+  /*std::cout << "Extracted weights size: " << weights.size() << std::endl;
   std::cout << "Shape: ";
   for (auto dim : shape) std::cout << dim << " ";
   std::cout << std::endl;
-  std::cout << "Extracted bias size: " << bias.size() << std::endl;
+  std::cout << "Extracted bias size: " << bias.size() << std::endl;*/
 
   return make_tensor<float>(weights, Shape(shape), bias);
 }