Fix lots of bugs due to newer kernel 5.8 in Ubuntu 20.04, and OpenZFS v0.8:

1. clone_extents_to() didn't handle partial clones, and now it does.

2. OpenZFS has a bug in SEEK_DATA which caused empty clones, we use a hacky
workaround to fix it.

3. mapped_file_handle now fails loudly if it fails to allocate the map due
to VMA exhaustion. This closes off a ton of unpleasant surprise in heavily
loaded applications.

4. If your Linux kernel was old enough, or you are on ZFS, renaming a
mapped_file_handle caused internal state corruption which led to lots of
bad things, including `bad_file_descriptor` errors.

5. On Windows, renaming a mapped_file_handle now tears down and restores
the maps, as you cannot rename a file with open maps.

Author: ned14

include/llfio/revision.hpp

@@ -1,4 +1,4 @@
 // Note the second line of this file must ALWAYS be the git SHA, third line ALWAYS the git SHA update time
-#define LLFIO_PREVIOUS_COMMIT_REF    74e710643944764a3548847322d6cb9a4a7ed5ab
-#define LLFIO_PREVIOUS_COMMIT_DATE   "2021-02-23 11:27:11 +00:00"
-#define LLFIO_PREVIOUS_COMMIT_UNIQUE 74e71064
+#define LLFIO_PREVIOUS_COMMIT_REF    a74411eddb6401ab884c5f92cccc24b9a64a9e6f
+#define LLFIO_PREVIOUS_COMMIT_DATE   "2021-02-23 14:33:57 +00:00"
+#define LLFIO_PREVIOUS_COMMIT_UNIQUE a74411ed

include/llfio/v2.0/detail/impl/posix/file_handle.ipp

@@ -581,6 +581,18 @@ result<file_handle::extent_pair> file_handle::clone_extents_to(file_handle::exte
       for(;;)
       {
 #ifdef __linux__
+        if(1)
+        {
+          /* There is a bug in ZFSOnLinux where SEEK_DATA returns no data in file
+          if the file was rewritten using a mmap which is then closed, and the file
+          immediately reopened and SEEK_DATA called upon it. If you read a single
+          byte, it seems to kick SEEK_DATA into working correctly.
+
+          See https://github.com/openzfs/zfs/issues/11697 for more.
+          */
+          char b;
+          (void) ::pread(_v.fd, &b, 1, end);
+        }
         start = lseek64(_v.fd, end, SEEK_DATA);
 #else
         start = lseek(_v.fd, end, SEEK_DATA);
@@ -747,17 +759,17 @@ result<file_handle::extent_pair> file_handle::clone_extents_to(file_handle::exte
     auto _copy_file_range = [&](int infd, off_t *inoffp, int outfd, off_t *outoffp, size_t len, unsigned int flags) -> ssize_t {
 #if defined(__linux__)
 #if defined __aarch64__
-       return syscall(285 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
+      return syscall(285 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
 #elif defined __arm__
-       return syscall(391 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
+      return syscall(391 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
 #elif defined __i386__
-       return syscall(377 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
+      return syscall(377 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
 #elif defined __powerpc64__
-       return syscall(379 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
+      return syscall(379 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
 #elif defined __sparc__
-       return syscall(357 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
+      return syscall(357 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
 #elif defined __x86_64__
-       return syscall(326 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
+      return syscall(326 /*__NR_copy_file_range*/, infd, inoffp, outfd, outoffp, len, flags);
 #else
 #error Unknown Linux platform
 #endif
@@ -790,23 +802,30 @@ result<file_handle::extent_pair> file_handle::clone_extents_to(file_handle::exte
     {
       for(extent_type thisoffset = 0; thisoffset < item.src.length; thisoffset += blocksize)
       {
+      retry_clone:
         bool done = false;
         const auto thisblock = std::min(blocksize, item.src.length - thisoffset);
         if(duplicate_extents && item.op == workitem::clone_extents)
         {
           off_t off_in = item.src.offset + thisoffset, off_out = item.src.offset + thisoffset + destoffsetdiff;
-          if(_copy_file_range(_v.fd, &off_in, dest.native_handle().fd, &off_out, thisblock, 0) < 0)
+          auto bytes_cloned = _copy_file_range(_v.fd, &off_in, dest.native_handle().fd, &off_out, thisblock, 0);
+          if(bytes_cloned <= 0)
           {
-            if((EXDEV != errno && EOPNOTSUPP != errno && ENOSYS != errno) || !emulate_if_unsupported)
+            if(bytes_cloned < 0 && ((EXDEV != errno && EOPNOTSUPP != errno && ENOSYS != errno) || !emulate_if_unsupported))
             {
               return posix_error();
             }
             duplicate_extents = false;  // emulate using copy of bytes
           }
-          else
+          else if((size_t) bytes_cloned == thisblock)
           {
             done = true;
           }
+          else
+          {
+            thisoffset += bytes_cloned;
+            goto retry_clone;
+          }
         }
         if(!done && (item.op == workitem::copy_bytes || (!duplicate_extents && item.op == workitem::clone_extents)))
         {

include/llfio/v2.0/detail/impl/posix/mapped_file_handle.ipp

@@ -1,5 +1,5 @@
 /* An mapped handle to a file
-(C) 2017 Niall Douglas <http://www.nedproductions.biz/> (11 commits)
+(C) 2017-2021 Niall Douglas <http://www.nedproductions.biz/> (11 commits)
 File Created: Sept 2017
 
 
@@ -27,15 +27,20 @@ Distributed under the Boost Software License, Version 1.0.
 
 LLFIO_V2_NAMESPACE_BEGIN
 
-result<mapped_file_handle::size_type> mapped_file_handle::reserve(size_type reservation) noexcept
+result<mapped_file_handle::size_type> mapped_file_handle::_reserve(extent_type &length, size_type reservation) noexcept
 {
   LLFIO_LOG_FUNCTION_CALL(this);
-  OUTCOME_TRY(auto &&length, underlying_file_maximum_extent());
+#ifndef NDEBUG
+  if(_mh.is_valid())
+  {
+    assert(_mh.native_handle()._init == native_handle()._init);
+  }
+#endif
+  OUTCOME_TRY(length, underlying_file_maximum_extent());
   if(length == 0)
   {
-    // Not portable to map an empty file, so do nothing
+    // Not portable to map an empty file, but retain reservation
     _reservation = reservation;
-    return success();
   }
   if(reservation == 0)
   {
@@ -70,6 +75,7 @@ result<void> mapped_file_handle::close() noexcept
   LLFIO_LOG_FUNCTION_CALL(this);
   if(_mh.is_valid())
   {
+    assert(_mh.native_handle()._init == native_handle()._init);
     OUTCOME_TRYV(_mh.close());
   }
   if(_sh.is_valid())
@@ -83,6 +89,7 @@ native_handle_type mapped_file_handle::release() noexcept
   LLFIO_LOG_FUNCTION_CALL(this);
   if(_mh.is_valid())
   {
+    assert(_mh.native_handle()._init == native_handle()._init);
     (void) _mh.close();
   }
   if(_sh.is_valid())
@@ -95,6 +102,12 @@ native_handle_type mapped_file_handle::release() noexcept
 result<mapped_file_handle::extent_type> mapped_file_handle::truncate(extent_type newsize) noexcept
 {
   LLFIO_LOG_FUNCTION_CALL(this);
+#ifndef NDEBUG
+  if(_mh.is_valid())
+  {
+    assert(_mh.native_handle()._init == native_handle()._init);
+  }
+#endif
   // Release all maps and sections and truncate the backing file to zero
   if(newsize == 0)
   {
@@ -142,6 +155,12 @@ result<mapped_file_handle::extent_type> mapped_file_handle::truncate(extent_type
 
 result<mapped_file_handle::extent_type> mapped_file_handle::update_map() noexcept
 {
+#ifndef NDEBUG
+  if(_mh.is_valid())
+  {
+    assert(_mh.native_handle()._init == native_handle()._init);
+  }
+#endif
   OUTCOME_TRY(auto &&length, underlying_file_maximum_extent());
   if(length > _reservation)
   {
@@ -156,12 +175,47 @@ result<mapped_file_handle::extent_type> mapped_file_handle::update_map() noexcep
   }
   if(!_sh.is_valid())
   {
-    OUTCOME_TRYV(reserve(_reservation));
+    (void) reserve(_reservation);
     return length;
   }
   // Adjust the map to reflect the new size of the section
   _mh._length = length;
   return length;
 }
 
+result<void> mapped_file_handle::relink(const path_handle &base, path_view_type path, bool atomic_replace, deadline d) noexcept
+{
+#ifndef NDEBUG
+  if(_mh.is_valid())
+  {
+    assert(_mh.native_handle()._init == native_handle()._init);
+  }
+#endif
+  // On POSIX relink() may change _v.fd sometimes.
+  auto ret = file_handle::relink(base, path, atomic_replace, d);
+  bool restore_maps = false;
+  if(_sh.is_valid() && _v.fd != _sh.native_handle().fd)
+  {
+    OUTCOME_TRY(_sh.close());
+    restore_maps = true;
+  }
+  if(_mh.is_valid() && _v.fd != _mh.native_handle().fd)
+  {
+    OUTCOME_TRY(_mh.close());
+    restore_maps = true;
+  }
+  if(restore_maps)
+  {
+    OUTCOME_TRY(reserve(_reservation));
+  }
+#ifndef NDEBUG
+  if(_mh.is_valid())
+  {
+    assert(_mh.native_handle()._init == native_handle()._init);
+  }
+#endif
+  return ret;
+}
+
+
 LLFIO_V2_NAMESPACE_END

include/llfio/v2.0/detail/impl/windows/mapped_file_handle.ipp

@@ -1,5 +1,5 @@
 /* An mapped handle to a file
-(C) 2017 Niall Douglas <http://www.nedproductions.biz/> (11 commits)
+(C) 2017-2021 Niall Douglas <http://www.nedproductions.biz/> (11 commits)
 File Created: Sept 2017
 
 
@@ -27,15 +27,20 @@ Distributed under the Boost Software License, Version 1.0.
 
 LLFIO_V2_NAMESPACE_BEGIN
 
-result<mapped_file_handle::size_type> mapped_file_handle::reserve(size_type reservation) noexcept
+result<mapped_file_handle::size_type> mapped_file_handle::_reserve(extent_type &length, size_type reservation) noexcept
 {
   LLFIO_LOG_FUNCTION_CALL(this);
-  OUTCOME_TRY(auto &&length, underlying_file_maximum_extent());
+#ifndef NDEBUG
+  if(_mh.is_valid())
+  {
+    assert(_mh.native_handle()._init == native_handle()._init);
+  }
+#endif
+  OUTCOME_TRY(length, underlying_file_maximum_extent());
   if(length == 0)
   {
-    // Cannot create a section nor a map for a zero lengthed file
+    // Not portable to map an empty file, but retain reservation
     _reservation = reservation;
-    return success();
   }
   if(reservation == 0)
   {
@@ -80,6 +85,7 @@ result<void> mapped_file_handle::close() noexcept
   LLFIO_LOG_FUNCTION_CALL(this);
   if(_mh.is_valid())
   {
+    assert(_mh.native_handle()._init == native_handle()._init);
     OUTCOME_TRYV(_mh.close());
   }
   if(_sh.is_valid())
@@ -93,6 +99,7 @@ native_handle_type mapped_file_handle::release() noexcept
   LLFIO_LOG_FUNCTION_CALL(this);
   if(_mh.is_valid())
   {
+    assert(_mh.native_handle()._init == native_handle()._init);
     (void) _mh.close();
   }
   if(_sh.is_valid())
@@ -105,6 +112,12 @@ native_handle_type mapped_file_handle::release() noexcept
 result<mapped_file_handle::extent_type> mapped_file_handle::truncate(extent_type newsize) noexcept
 {
   LLFIO_LOG_FUNCTION_CALL(this);
+#ifndef NDEBUG
+  if(_mh.is_valid())
+  {
+    assert(_mh.native_handle()._init == native_handle()._init);
+  }
+#endif
   // Release all maps and sections and truncate the backing file to zero
   if(newsize == 0)
   {
@@ -161,6 +174,12 @@ result<mapped_file_handle::extent_type> mapped_file_handle::truncate(extent_type
 
 result<mapped_file_handle::extent_type> mapped_file_handle::update_map() noexcept
 {
+#ifndef NDEBUG
+  if(_mh.is_valid())
+  {
+    assert(_mh.native_handle()._init == native_handle()._init);
+  }
+#endif
   OUTCOME_TRY(auto &&length, underlying_file_maximum_extent());
   if(length > _reservation)
   {
@@ -175,7 +194,7 @@ result<mapped_file_handle::extent_type> mapped_file_handle::update_map() noexcep
   }
   if(!_sh.is_valid())
   {
-    OUTCOME_TRYV(reserve(_reservation));
+    (void) reserve(_reservation);
     return length;
   }
   OUTCOME_TRY(auto &&size, _sh.length());
@@ -193,4 +212,38 @@ result<mapped_file_handle::extent_type> mapped_file_handle::update_map() noexcep
   return length;
 }
 
+result<void> mapped_file_handle::relink(const path_handle &base, path_view_type path, bool atomic_replace, deadline d) noexcept
+{
+#ifndef NDEBUG
+  if(_mh.is_valid())
+  {
+    assert(_mh.native_handle()._init == native_handle()._init);
+  }
+#endif
+  // On Windows relink() won't succeed unless all maps are torn down
+  bool restore_maps = false;
+  if(_sh.is_valid())
+  {
+    OUTCOME_TRY(_sh.close());
+    restore_maps = true;
+  }
+  if(_mh.is_valid())
+  {
+    OUTCOME_TRY(_mh.close());
+    restore_maps = true;
+  }
+  auto ret = file_handle::relink(base, path, atomic_replace, d);
+  if(restore_maps)
+  {
+    OUTCOME_TRY(reserve(_reservation));
+  }
+#ifndef NDEBUG
+  if(_mh.is_valid())
+  {
+    assert(_mh.native_handle()._init == native_handle()._init);
+  }
+#endif
+  return ret;
+}
+
 LLFIO_V2_NAMESPACE_END