Merge pull request #771 from PowerGridModel/feature/columnar-data-sup…

…port-no-id-field

Feature / columnar data optional id field support in update data

power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp

@@ -171,7 +171,7 @@ template <dataset_type_tag dataset_type_> class Dataset {
     template <class StructType>
     using DataStruct = std::conditional_t<is_data_mutable_v<dataset_type>, StructType, StructType const>;
 
-    // for columnar buffers, Data* data is empty and attributes is filled
+    // for columnar buffers, Data* data is empty and attributes.data is filled
     // for uniform buffers, indptr is empty
     struct Buffer {
         using Data = Dataset::Data;
@@ -227,10 +227,14 @@ template <dataset_type_tag dataset_type_> class Dataset {
     MetaDataset const& dataset() const { return *dataset_info_.dataset; }
     Idx n_components() const { return static_cast<Idx>(buffers_.size()); }
     DatasetInfo const& get_description() const { return dataset_info_; }
-    ComponentInfo const& get_component_info(Idx i) const { return dataset_info_.component_info[i]; }
     Buffer const& get_buffer(std::string_view component) const { return get_buffer(find_component(component, true)); }
     Buffer const& get_buffer(Idx i) const { return buffers_[i]; }
 
+    ComponentInfo const& get_component_info(std::string_view component) const {
+        return get_component_info(find_component(component, true));
+    }
+    ComponentInfo const& get_component_info(Idx i) const { return dataset_info_.component_info[i]; }
+
     constexpr bool is_row_based(std::string_view component) const {
         Idx const idx = find_component(component, false);
         if (idx == invalid_index) {
@@ -254,6 +258,63 @@ template <dataset_type_tag dataset_type_> class Dataset {
         return !is_row_based(buffer) && !(with_attribute_buffers && buffer.attributes.empty());
     }
 
+    constexpr bool is_dense(std::string_view component) const {
+        Idx const idx = find_component(component, false);
+        if (idx == invalid_index) {
+            return true; // by definition
+        }
+        return is_dense(idx);
+    }
+    constexpr bool is_dense(Idx const i) const { return is_dense(buffers_[i]); }
+    constexpr bool is_dense(Buffer const& buffer) const { return buffer.indptr.empty(); }
+    constexpr bool is_sparse(std::string_view component, bool with_attribute_buffers = false) const {
+        Idx const idx = find_component(component, false);
+        if (idx == invalid_index) {
+            return false;
+        }
+        return is_sparse(idx, with_attribute_buffers);
+    }
+    constexpr bool is_sparse(Idx const i, bool with_attribute_buffers = false) const {
+        return is_sparse(buffers_[i], with_attribute_buffers);
+    }
+    constexpr bool is_sparse(Buffer const& buffer) const { return !is_dense(buffer); }
+
+    constexpr bool is_uniform(std::string_view component) const {
+        Idx const idx = find_component(component, false);
+        if (idx == invalid_index) {
+            return true; // by definition
+        }
+        return is_uniform(idx);
+    }
+    constexpr bool is_uniform(Idx const i) const { return is_uniform(buffers_[i]); }
+    constexpr bool is_uniform(Buffer const& buffer) const {
+        if (is_dense(buffer)) {
+            return true;
+        }
+        assert(buffer.indptr.size() > 1);
+        auto const first_scenario_size = buffer.indptr[1] - buffer.indptr[0];
+        return std::ranges::adjacent_find(buffer.indptr, [first_scenario_size](Idx start, Idx stop) {
+                   return stop - start != first_scenario_size;
+               }) == buffer.indptr.end();
+    }
+
+    constexpr Idx uniform_elements_per_scenario(std::string_view component) const {
+        Idx const idx = find_component(component, false);
+        if (idx == invalid_index) {
+            return 0;
+        }
+        return uniform_elements_per_scenario(idx);
+    }
+    constexpr Idx uniform_elements_per_scenario(Idx const i) const {
+        assert(is_uniform(i));
+        if (is_dense(i)) {
+            return get_component_info(i).elements_per_scenario;
+        }
+        auto const& indptr = buffers_[i].indptr;
+        assert(indptr.size() > 1);
+        return indptr[1] - indptr[0];
+    }
+
     Idx find_component(std::string_view component, bool required = false) const {
         auto const found = std::ranges::find_if(dataset_info_.component_info, [component](ComponentInfo const& x) {
             return x.component->name == component;
@@ -269,10 +330,6 @@ template <dataset_type_tag dataset_type_> class Dataset {
     }
     bool contains_component(std::string_view component) const { return find_component(component) >= 0; }
 
-    ComponentInfo const& get_component_info(std::string_view component) const {
-        return dataset_info_.component_info[find_component(component, true)];
-    }
-
     void add_component_info(std::string_view component, Idx elements_per_scenario, Idx total_elements)
         requires is_indptr_mutable_v<dataset_type>
     {

power_grid_model_c/power_grid_model/include/power_grid_model/common/common.hpp

@@ -66,6 +66,7 @@ constexpr double numerical_tolerance = 1e-8;
 constexpr double nan = std::numeric_limits<double>::quiet_NaN();
 constexpr IntS na_IntS = std::numeric_limits<IntS>::min();
 constexpr ID na_IntID = std::numeric_limits<ID>::min();
+constexpr Idx na_Idx = std::numeric_limits<Idx>::min();
 
 // power grid constant
 constexpr double base_power_3p = 1e6;

power_grid_model_c/power_grid_model/include/power_grid_model/common/exception.hpp

@@ -141,6 +141,13 @@ class IDNotFound : public PowerGridError {
   public:
     explicit IDNotFound(ID id) { append_msg("The id cannot be found: " + detail::to_string(id) + '\n'); }
 };
+class Idx2DNotFound : public PowerGridError {
+  public:
+    explicit Idx2DNotFound(Idx2D id) {
+        append_msg("The idx 2d cannot be found: {" + detail::to_string(id.group) + ", " + detail::to_string(id.pos) +
+                   "}.\n");
+    }
+};
 
 class InvalidMeasuredObject : public PowerGridError {
   public:

power_grid_model_c/power_grid_model/include/power_grid_model/common/three_phase_tensor.hpp

@@ -297,6 +297,7 @@ template <class Enum>
 inline bool is_nan(Enum x) {
     return static_cast<IntS>(x) == na_IntS;
 }
+inline bool is_nan(Idx x) { return x == na_Idx; }
 
 // is normal
 inline auto is_normal(std::floating_point auto value) { return std::isnormal(value); }

power_grid_model_c/power_grid_model/include/power_grid_model/component/branch.hpp

@@ -184,14 +184,14 @@ class Branch : public Base {
 
     // default update for branch, will be hidden for transformer
     UpdateChange update(BranchUpdate const& update_data) {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         bool const changed = set_status(update_data.from_status, update_data.to_status);
         // change branch connection will change both topo and param
         return {changed, changed};
     }
 
     auto inverse(std::convertible_to<BranchUpdate> auto update_data) const {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
 
         set_if_not_nan(update_data.from_status, static_cast<IntS>(from_status_));
         set_if_not_nan(update_data.to_status, static_cast<IntS>(to_status_));

power_grid_model_c/power_grid_model/include/power_grid_model/component/branch3.hpp

@@ -206,14 +206,14 @@ class Branch3 : public Base {
 
     // default update for branch3, will be hidden for three winding transformer
     UpdateChange update(Branch3Update const& update_data) {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         bool const changed = set_status(update_data.status_1, update_data.status_2, update_data.status_3);
         // change in branch3 connection will change both topo and param
         return {changed, changed};
     }
 
     auto inverse(std::convertible_to<Branch3Update> auto update_data) const {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
 
         set_if_not_nan(update_data.status_1, static_cast<IntS>(status_1_));
         set_if_not_nan(update_data.status_2, static_cast<IntS>(status_2_));

power_grid_model_c/power_grid_model/include/power_grid_model/component/fault.hpp

@@ -118,7 +118,7 @@ class Fault final : public Base {
     }
 
     FaultUpdate inverse(FaultUpdate update_data) const {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
 
         set_if_not_nan(update_data.status, static_cast<IntS>(status_));
         set_if_not_nan(update_data.fault_type, fault_type_);

power_grid_model_c/power_grid_model/include/power_grid_model/component/load_gen.hpp

@@ -102,18 +102,17 @@ class LoadGen final : public std::conditional_t<is_generator_v<appliance_type_>,
 
     // update for load_gen
     UpdateChange update(LoadGenUpdate<loadgen_symmetry> const& update_data) {
-        assert(update_data.id == this->id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         this->set_status(update_data.status);
         set_power(update_data.p_specified, update_data.q_specified);
         // change load connection and/or value will not change topology or parameters
         return {false, false};
     }
 
     LoadGenUpdate<loadgen_symmetry> inverse(LoadGenUpdate<loadgen_symmetry> update_data) const {
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         double const scalar = direction_ * base_power<loadgen_symmetry>;
 
-        assert(update_data.id == this->id());
-
         set_if_not_nan(update_data.status, static_cast<IntS>(this->status()));
         set_if_not_nan(update_data.p_specified, real(s_specified_) * scalar);
         set_if_not_nan(update_data.q_specified, imag(s_specified_) * scalar);

power_grid_model_c/power_grid_model/include/power_grid_model/component/power_sensor.hpp

@@ -99,8 +99,7 @@ template <symmetry_tag power_sensor_symmetry_> class PowerSensor : public Generi
     }
 
     PowerSensorUpdate<power_sensor_symmetry> inverse(PowerSensorUpdate<power_sensor_symmetry> update_data) const {
-        assert(update_data.id == this->id());
-
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         auto const scalar = convert_direction() * base_power<power_sensor_symmetry>;
 
         set_if_not_nan(update_data.p_measured, real(s_measured_) * scalar);

power_grid_model_c/power_grid_model/include/power_grid_model/component/regulator.hpp

@@ -29,7 +29,7 @@ class Regulator : public Base {
     void set_status(IntS status) { status_ = static_cast<bool>(status); }
 
     auto inverse(std::convertible_to<RegulatorUpdate> auto update_data) const {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         set_if_not_nan(update_data.status, static_cast<IntS>(status_));
         return update_data;
     }

power_grid_model_c/power_grid_model/include/power_grid_model/component/shunt.hpp

@@ -46,7 +46,7 @@ class Shunt : public Appliance {
     }
 
     UpdateChange update(ShuntUpdate const& update_data) {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         bool changed = set_status(update_data.status);
         changed = update_params(update_data) || changed;
 
@@ -55,7 +55,7 @@ class Shunt : public Appliance {
     }
 
     ShuntUpdate inverse(ShuntUpdate update_data) const {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
 
         set_if_not_nan(update_data.status, static_cast<IntS>(this->status()));
         set_if_not_nan(update_data.g1, g1_);

power_grid_model_c/power_grid_model/include/power_grid_model/component/source.hpp

@@ -88,7 +88,7 @@ class Source : public Appliance {
 
     // update for source
     UpdateChange update(SourceUpdate const& update_data) {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         bool const topo_changed = set_status(update_data.status);
         bool const param_changed = set_u_ref(update_data.u_ref, update_data.u_ref_angle);
         // change source connection will change both topo and param
@@ -97,7 +97,7 @@ class Source : public Appliance {
     }
 
     SourceUpdate inverse(SourceUpdate update_data) const {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
 
         set_if_not_nan(update_data.status, static_cast<IntS>(this->status()));
         set_if_not_nan(update_data.u_ref, u_ref_);

power_grid_model_c/power_grid_model/include/power_grid_model/component/three_winding_transformer.hpp

@@ -130,14 +130,14 @@ class ThreeWindingTransformer : public Branch3 {
     }
 
     UpdateChange update(ThreeWindingTransformerUpdate const& update_data) {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         bool const topo_changed = set_status(update_data.status_1, update_data.status_2, update_data.status_3);
         bool const param_changed = set_tap(update_data.tap_pos) || topo_changed;
         return {topo_changed, param_changed};
     }
 
     ThreeWindingTransformerUpdate inverse(ThreeWindingTransformerUpdate update_data) const {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
 
         update_data = Branch3::inverse(update_data);
         set_if_not_nan(update_data.tap_pos, tap_pos_);

power_grid_model_c/power_grid_model/include/power_grid_model/component/transformer.hpp

@@ -94,14 +94,14 @@ class Transformer : public Branch {
 
     // update for transformer, hide default update for branch
     UpdateChange update(TransformerUpdate const& update_data) {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         bool const topo_changed = set_status(update_data.from_status, update_data.to_status);
         bool const param_changed = set_tap(update_data.tap_pos) || topo_changed;
         return {topo_changed, param_changed};
     }
 
     TransformerUpdate inverse(TransformerUpdate update_data) const {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
 
         update_data = Branch::inverse(update_data);
         set_if_not_nan(update_data.tap_pos, tap_pos_);

power_grid_model_c/power_grid_model/include/power_grid_model/component/transformer_tap_regulator.hpp

@@ -35,7 +35,7 @@ class TransformerTapRegulator : public Regulator {
 
     // update for transformer tap regulator, hide default update for branch
     UpdateChange update(TransformerTapRegulatorUpdate const& update_data) {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         set_status(update_data.status);
         update_real_value<symmetric_t>(update_data.u_set, u_set_, 1.0);
         update_real_value<symmetric_t>(update_data.u_band, u_band_, 1.0);
@@ -45,7 +45,7 @@ class TransformerTapRegulator : public Regulator {
     }
 
     TransformerTapRegulatorUpdate inverse(TransformerTapRegulatorUpdate update_data) const {
-        assert(update_data.id == id());
+        assert(update_data.id == this->id() || is_nan(update_data.id));
 
         update_data = Regulator::inverse(update_data);
         set_if_not_nan(update_data.u_set, u_set_);

power_grid_model_c/power_grid_model/include/power_grid_model/component/voltage_sensor.hpp

@@ -80,8 +80,7 @@ template <symmetry_tag sym> class VoltageSensor : public GenericVoltageSensor {
           u_angle_measured_{voltage_sensor_input.u_angle_measured} {};
 
     UpdateChange update(VoltageSensorUpdate<sym> const& update_data) {
-        assert(update_data.id == this->id());
-
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         double const scalar = 1 / (u_rated_ * u_scale<sym>);
 
         update_real_value<sym>(update_data.u_measured, u_measured_, scalar);
@@ -95,8 +94,7 @@ template <symmetry_tag sym> class VoltageSensor : public GenericVoltageSensor {
     }
 
     VoltageSensorUpdate<sym> inverse(VoltageSensorUpdate<sym> update_data) const {
-        assert(update_data.id == this->id());
-
+        assert(update_data.id == this->id() || is_nan(update_data.id));
         double const scalar = u_rated_ * u_scale<sym>;
 
         set_if_not_nan(update_data.u_measured, u_measured_ * scalar);

power_grid_model_c/power_grid_model/include/power_grid_model/container.hpp

@@ -110,6 +110,17 @@ class Container<RetrievableTypes<GettableTypes...>, StorageableTypes...> {
         assert(is_base<Gettable>[idx_2d.group]);
         return (this->*(func_arr[idx_2d.group]))(idx_2d.pos);
     }
+
+#ifndef NDEBUG
+    // get id by idx, only for debugging purpose
+    ID get_id_by_idx(Idx2D idx_2d) const {
+        if (auto it = std::ranges::find(map_, idx_2d, &std::pair<const ID, Idx2D>::second); it != map_.end()) {
+            return it->first;
+        }
+        throw Idx2DNotFound{idx_2d};
+    }
+#endif // NDEBUG
+
     // get idx by id
     Idx2D get_idx_by_id(ID id) const {
         auto const found = map_.find(id);
@@ -125,6 +136,10 @@ class Container<RetrievableTypes<GettableTypes...>, StorageableTypes...> {
         }
         return result;
     }
+    template <supported_type_c<StorageableTypes...> Storageable> constexpr Idx get_group_idx() const {
+        return static_cast<Idx>(get_cls_pos_v<Storageable, StorageableTypes...>);
+    }
+
     // get item based on ID
     template <supported_type_c<GettableTypes...> Gettable> Gettable& get_item(ID id) {
         Idx2D const idx = get_idx_by_id<Gettable>(id);

power_grid_model_c/power_grid_model/include/power_grid_model/main_core/state_queries.hpp

@@ -33,6 +33,12 @@ inline Idx2D get_component_idx_by_id(MainModelState<ComponentContainer> const& s
     return state.components.template get_idx_by_id<ComponentType>(id);
 }
 
+template <typename ComponentType, class ComponentContainer>
+    requires model_component_state_c<MainModelState, ComponentContainer, ComponentType>
+constexpr Idx get_component_group_idx(MainModelState<ComponentContainer> const& state) {
+    return state.components.template get_group_idx<ComponentType>();
+}
+
 template <typename ComponentType, class ComponentContainer>
     requires model_component_state_c<MainModelState, ComponentContainer, ComponentType>
 inline Idx get_component_sequence(MainModelState<ComponentContainer> const& state, auto const& id_or_index) {