[ntuple] automatic evolution from fixed-size array to RVec

tree/dataframe/src/RNTupleDS.cxx

@@ -197,15 +197,20 @@ class RNTupleColumnReader : public ROOT::Detail::RDF::RColumnReaderBase {
          }
       }
 
+      RFieldZero fieldZero;
+      ROOT::Internal::SetAllowFieldSubstitutions(fieldZero, true);
+      fieldZero.Attach(std::move(fField));
       try {
-         ROOT::Internal::CallConnectPageSourceOnField(*fField, source);
+         ROOT::Internal::CallConnectPageSourceOnField(fieldZero, source);
       } catch (const ROOT::RException &err) {
+         fField = std::move(fieldZero.ReleaseSubfields()[0]);
          auto onDiskType = source.GetSharedDescriptorGuard()->GetFieldDescriptor(fField->GetOnDiskId()).GetTypeName();
          std::string msg = "RNTupleDS: invalid type \"" + fField->GetTypeName() + "\" for column \"" +
                            fDataSource->fFieldId2QualifiedName[fField->GetOnDiskId()] + "\" with on-disk type \"" +
                            onDiskType + "\"";
          throw std::runtime_error(msg);
       }
+      fField = std::move(fieldZero.ReleaseSubfields()[0]);
 
       if (fValuePtr) {
          // When the reader reconnects to a new file, the fValuePtr is already set

tree/ntuple/inc/ROOT/RField.hxx

@@ -49,9 +49,26 @@ namespace Detail {
 class RFieldVisitor;
 } // namespace Detail
 
+class RFieldZero;
+namespace Internal {
+void SetAllowFieldSubstitutions(RFieldZero &fieldZero, bool val);
+}
+
 /// The container field for an ntuple model, which itself has no physical representation.
 /// Therefore, the zero field must not be connected to a page source or sink.
 class RFieldZero final : public RFieldBase {
+   friend void ROOT::Internal::SetAllowFieldSubstitutions(RFieldZero &, bool);
+
+   /// If field substitutions are allowed, upon connecting to a page source the field hierarchy will replace created
+   /// fields by fields that match the on-disk schema. This happens for
+   ///     - Vector fields (RVectorField, RRVecField) that connect to an on-disk fixed-size array
+   ///     - Streamer fields that connect to an on-disk class field
+   /// Field substitutions must not be enabled when the field hierarchy already handed out RValue objects because
+   /// they would leave dangling field pointers to the replaced fields. This is used in cases when the field/model
+   /// is created by RNTuple (not imposed), before it is made available to the user.
+   /// This flag is reset on Clone().
+   bool fAllowFieldSubstitutions = false;
+
 protected:
    std::unique_ptr<RFieldBase> CloneImpl(std::string_view newName) const final;
    void ConstructValue(void *) const final {}
@@ -64,6 +81,10 @@ public:
    size_t GetAlignment() const final { return 0; }
 
    void AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) const final;
+
+   bool GetAllowFieldSubstitutions() const { return fAllowFieldSubstitutions; }
+   /// Moves all subfields into the returned vector.
+   std::vector<std::unique_ptr<RFieldBase>> ReleaseSubfields();
 };
 
 /// Used in RFieldBase::Check() to record field creation failures.
@@ -187,7 +208,7 @@ protected:
    void ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void *to) final;
    void ReadInClusterImpl(RNTupleLocalIndex localIndex, void *to) final;
 
-   void BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) final;
+   std::unique_ptr<RFieldBase> BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) final;
    void ReconcileOnDiskField(const RNTupleDescriptor &desc) final;
 
 public:
@@ -242,7 +263,7 @@ protected:
    // Returns the list of seen streamer infos
    ROOT::RExtraTypeInfoDescriptor GetExtraTypeInfo() const final;
 
-   void BeforeConnectPageSource(ROOT::Internal::RPageSource &source) final;
+   std::unique_ptr<RFieldBase> BeforeConnectPageSource(ROOT::Internal::RPageSource &source) final;
    void ReconcileOnDiskField(const RNTupleDescriptor &desc) final;
 
 public:

tree/ntuple/inc/ROOT/RField/RFieldSequenceContainer.hxx

@@ -111,6 +111,13 @@ public:
 
 /// The type-erased field for a RVec<Type>
 class RRVecField : public RFieldBase {
+   friend class RArrayAsRVecField; // to call ResizeRVec()
+
+   // Ensures that the RVec pointed to by rvec has at least nItems valid elements
+   // Returns the possibly new "begin pointer" of the RVec, i.e. the pointer to the data area.
+   static unsigned char *
+   ResizeRVec(void *rvec, std::size_t nItems, std::size_t itemSize, const RFieldBase *itemField, RDeleter *itemDeleter);
+
 public:
    /// the RRVecDeleter is also used by RArrayAsRVecField and therefore declared public
    class RRVecDeleter : public RDeleter {
@@ -147,6 +154,7 @@ protected:
    void ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void *to) final;
    std::size_t ReadBulkImpl(const RBulkSpec &bulkSpec) final;
 
+   std::unique_ptr<RFieldBase> BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource) final;
    void ReconcileOnDiskField(const RNTupleDescriptor &desc) final;
 
    void CommitClusterImpl() final { fNWritten = 0; }

tree/ntuple/inc/ROOT/RFieldBase.hxx

@@ -85,6 +85,7 @@ This is and can only be partially enforced through C++.
 */
 // clang-format on
 class RFieldBase {
+   friend class RFieldZero;                                    // to reset fParent pointer in ReleaseSubfields()
    friend class ROOT::Experimental::Detail::RRawPtrWriteEntry; // to call Append()
    friend struct ROOT::Internal::RFieldCallbackInjector;       // used for unit tests
    friend struct ROOT::Internal::RFieldRepresentationModifier; // used for unit tests
@@ -513,7 +514,13 @@ protected:
    /// Called by ConnectPageSource() before connecting; derived classes may override this as appropriate, e.g.
    /// for the application of I/O rules. In the process, the field at hand or its subfields may be marked as
    /// "artifical", i.e. introduced by schema evolution and not backed by on-disk information.
-   virtual void BeforeConnectPageSource(ROOT::Internal::RPageSource & /* source */) {}
+   /// May return a field substitute that fits the on-disk schema as a replacement for the field at hand.
+   /// A field substitute must read into the same in-memory layout than the original field and field substitutions
+   /// must not be cyclic.
+   virtual std::unique_ptr<RFieldBase> BeforeConnectPageSource(ROOT::Internal::RPageSource & /* source */)
+   {
+      return nullptr;
+   }
 
    /// For non-artificial fields, check compatibility of the in-memory field and the on-disk field. In the process,
    /// the field at hand may change its on-disk ID or perform other tasks related to automatic schema evolution.

tree/ntuple/inc/ROOT/RNTupleReader.hxx

@@ -93,7 +93,7 @@ private:
    /// The model is generated from the RNTuple metadata on storage.
    explicit RNTupleReader(std::unique_ptr<Internal::RPageSource> source, const ROOT::RNTupleReadOptions &options);
 
-   void ConnectModel(ROOT::RNTupleModel &model);
+   void ConnectModel(ROOT::RNTupleModel &model, bool allowFieldSubstitutions);
    RNTupleReader *GetDisplayReader();
    void InitPageSource(bool enableMetrics);
 

tree/ntuple/inc/ROOT/RNTupleView.hxx

@@ -89,6 +89,8 @@ protected:
    static std::unique_ptr<ROOT::RFieldBase>
    CreateField(ROOT::DescriptorId_t fieldId, Internal::RPageSource &pageSource, std::string_view typeName = "")
    {
+      RFieldZero fieldZero;
+      Internal::SetAllowFieldSubstitutions(fieldZero, true);
       std::unique_ptr<ROOT::RFieldBase> field;
       {
          const auto &desc = pageSource.GetSharedDescriptorGuard().GetRef();
@@ -103,8 +105,9 @@ protected:
          }
       }
       field->SetOnDiskId(fieldId);
-      ROOT::Internal::CallConnectPageSourceOnField(*field, pageSource);
-      return field;
+      fieldZero.Attach(std::move(field));
+      ROOT::Internal::CallConnectPageSourceOnField(fieldZero, pageSource);
+      return std::move(fieldZero.ReleaseSubfields()[0]);
    }
 
    RNTupleViewBase(std::unique_ptr<ROOT::RFieldBase> field, ROOT::RNTupleGlobalRange range)

tree/ntuple/src/RField.cxx

@@ -31,6 +31,11 @@
 #include <type_traits>
 #include <unordered_set>
 
+void ROOT::Internal::SetAllowFieldSubstitutions(RFieldZero &fieldZero, bool val)
+{
+   fieldZero.fAllowFieldSubstitutions = val;
+}
+
 std::unique_ptr<ROOT::RFieldBase> ROOT::RFieldZero::CloneImpl(std::string_view /*newName*/) const
 {
    auto result = std::make_unique<RFieldZero>();
@@ -39,6 +44,15 @@ std::unique_ptr<ROOT::RFieldBase> ROOT::RFieldZero::CloneImpl(std::string_view /
    return result;
 }
 
+std::vector<std::unique_ptr<ROOT::RFieldBase>> ROOT::RFieldZero::ReleaseSubfields()
+{
+   std::vector<std::unique_ptr<ROOT::RFieldBase>> result;
+   std::swap(fSubfields, result);
+   for (auto &f : result)
+      f->fParent = nullptr;
+   return result;
+}
+
 void ROOT::RFieldZero::AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) const
 {
    visitor.VisitFieldZero(*this);

tree/ntuple/src/RFieldBase.cxx

@@ -945,8 +945,12 @@ void ROOT::RFieldBase::ConnectPageSink(ROOT::Internal::RPageSink &pageSink, ROOT
 
 void ROOT::RFieldBase::ConnectPageSource(ROOT::Internal::RPageSource &pageSource)
 {
-   if (dynamic_cast<ROOT::RFieldZero *>(this))
-      throw RException(R__FAIL("invalid attempt to connect zero field to page source"));
+   if (dynamic_cast<ROOT::RFieldZero *>(this)) {
+      for (auto &f : fSubfields)
+         f->ConnectPageSource(pageSource);
+      return;
+   }
+
    if (fState != EState::kUnconnected)
       throw RException(R__FAIL("invalid attempt to connect an already connected field to a page source"));
 
@@ -955,7 +959,26 @@ void ROOT::RFieldBase::ConnectPageSource(ROOT::Internal::RPageSource &pageSource
    if (!fDescription.empty())
       throw RException(R__FAIL("setting description only valid when connecting to a page sink"));
 
-   BeforeConnectPageSource(pageSource);
+   auto substitute = BeforeConnectPageSource(pageSource);
+   if (substitute) {
+      const RFieldBase *itr = this;
+      while (itr->GetParent()) {
+         itr = itr->GetParent();
+      }
+      if (typeid(*itr) == typeid(RFieldZero) && static_cast<const RFieldZero *>(itr)->GetAllowFieldSubstitutions()) {
+         for (auto &f : fParent->fSubfields) {
+            if (f.get() != this)
+               continue;
+
+            f = std::move(substitute);
+            f->ConnectPageSource(pageSource);
+            return;
+         }
+         R__ASSERT(false); // never here
+      } else {
+         throw RException(R__FAIL("invalid attempt to substitute field " + GetQualifiedFieldName()));
+      }
+   }
 
    if (!fIsArtificial) {
       R__ASSERT(fOnDiskId != kInvalidDescriptorId);

tree/ntuple/src/RFieldMeta.cxx

@@ -393,17 +393,20 @@ void ROOT::RClassField::PrepareStagingArea(const std::vector<const TSchemaRule *
             throw RException(R__FAIL(std::string("cannot find on disk rule source member ") + GetTypeName() + "." +
                                      source->GetName()));
          }
-         const auto &memberFieldDesc = desc.GetFieldDescriptor(memberFieldId);
 
          auto memberType = source->GetTypeForDeclaration() + source->GetDimensions();
-         stagingItem.fField = Create("" /* we don't need a field name */, std::string(memberType)).Unwrap();
-         stagingItem.fField->SetOnDiskId(memberFieldDesc.GetId());
+         auto memberField = Create("" /* we don't need a field name */, std::string(memberType)).Unwrap();
+         memberField->SetOnDiskId(memberFieldId);
+         auto fieldZero = std::make_unique<RFieldZero>();
+         Internal::SetAllowFieldSubstitutions(*fieldZero, true);
+         fieldZero->Attach(std::move(memberField));
+         stagingItem.fField = std::move(fieldZero);
 
          stagingItem.fOffset = fStagingClass->GetDataMemberOffset(source->GetName());
          // Since we successfully looked up the source member in the RNTuple on-disk metadata, we expect it
          // to be present in the TClass instance, too.
          R__ASSERT(stagingItem.fOffset != TVirtualStreamerInfo::kMissing);
-         stagingAreaSize = std::max(stagingAreaSize, stagingItem.fOffset + stagingItem.fField->GetValueSize());
+         stagingAreaSize = std::max(stagingAreaSize, stagingItem.fOffset + stagingItem.fField->begin()->GetValueSize());
       }
    }
 
@@ -413,8 +416,9 @@ void ROOT::RClassField::PrepareStagingArea(const std::vector<const TSchemaRule *
       fStagingArea = std::make_unique<unsigned char[]>(stagingAreaSize);
 
       for (const auto &[_, si] : fStagingItems) {
-         if (!(si.fField->GetTraits() & kTraitTriviallyConstructible)) {
-            CallConstructValueOn(*si.fField, fStagingArea.get() + si.fOffset);
+         const auto &memberField = *si.fField->cbegin();
+         if (!(memberField.GetTraits() & kTraitTriviallyConstructible)) {
+            CallConstructValueOn(memberField, fStagingArea.get() + si.fOffset);
          }
       }
    }
@@ -436,7 +440,7 @@ void ROOT::RClassField::AddReadCallbacksFromIORule(const TSchemaRule *rule)
    });
 }
 
-void ROOT::RClassField::BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource)
+std::unique_ptr<ROOT::RFieldBase> ROOT::RClassField::BeforeConnectPageSource(ROOT::Internal::RPageSource &pageSource)
 {
    std::vector<const TSchemaRule *> rules;
    // On-disk members that are not targeted by an I/O rule; all other sub fields of the in-memory class
@@ -476,8 +480,10 @@ void ROOT::RClassField::BeforeConnectPageSource(ROOT::Internal::RPageSource &pag
       if (!rules.empty()) {
          SetStagingClass(fieldDesc.GetTypeName(), fieldDesc.GetTypeVersion());
          PrepareStagingArea(rules, desc, fieldDesc);
-         for (auto &[_, si] : fStagingItems)
+         for (auto &[_, si] : fStagingItems) {
             Internal::CallConnectPageSourceOnField(*si.fField, pageSource);
+            si.fField = std::move(static_cast<RFieldZero *>(si.fField.get())->ReleaseSubfields()[0]);
+         }
 
          // Remove target member of read rules from the list of regular members of the underlying on-disk field
          for (const auto rule : rules) {
@@ -501,6 +507,8 @@ void ROOT::RClassField::BeforeConnectPageSource(ROOT::Internal::RPageSource &pag
          CallSetArtificialOn(*field);
       }
    }
+
+   return nullptr;
 }
 
 void ROOT::RClassField::ReconcileOnDiskField(const RNTupleDescriptor &desc)
@@ -973,9 +981,10 @@ void ROOT::RStreamerField::GenerateColumns(const ROOT::RNTupleDescriptor &desc)
    GenerateColumnsImpl<ROOT::Internal::RColumnIndex, std::byte>(desc);
 }
 
-void ROOT::RStreamerField::BeforeConnectPageSource(ROOT::Internal::RPageSource &source)
+std::unique_ptr<ROOT::RFieldBase> ROOT::RStreamerField::BeforeConnectPageSource(ROOT::Internal::RPageSource &source)
 {
    source.RegisterStreamerInfos();
+   return nullptr;
 }
 
 void ROOT::RStreamerField::ReconcileOnDiskField(const RNTupleDescriptor &desc)