#14607: Index open performs version check on each segment, ignores indexCreatedVersionMajor

Author: rahulgoswami

lucene/CHANGES.txt

@@ -131,6 +131,8 @@ Other
 * GITHUB#14761: Use more Comparators for PriorityQueue implementations. (Simon Cooper)
 * GITHUB#14817: Refactor some complex uses of PriorityQueue to use Comparators. (Simon Cooper)
 
+* GITHUB#14607: Index open performs version check on each segment, ignores indexCreatedVersionMajor (Rahul Goswami)
+
 ======================= Lucene 10.4.0 =======================
 
 API Changes

lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java

@@ -45,6 +45,7 @@
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.tests.util.TestUtil;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.Version;
 
 public class TestAncientIndicesCompatibility extends LuceneTestCase {
   static final Set<String> UNSUPPORTED_INDEXES;
@@ -199,7 +200,7 @@ public void testUnsupportedOldIndexes() throws Exception {
       checker.setInfoStream(new PrintStream(bos, false, UTF_8));
       checker.setLevel(CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS);
       CheckIndex.Status indexStatus = checker.checkIndex();
-      if (version.startsWith("8.") || version.startsWith("9.")) {
+      if (getVersion(version).onOrAfter(Version.fromBits(8, 6, 0))) {
         assertTrue(indexStatus.clean);
       } else {
         assertFalse(indexStatus.clean);
@@ -217,6 +218,18 @@ public void testUnsupportedOldIndexes() throws Exception {
     }
   }
 
+  private Version getVersion(String version) {
+    if (version.startsWith("5x")) {
+      // couple of indices in unsupported_indices.txt start with "5x'
+      return Version.fromBits(5, 0, 0);
+    }
+    String[] versionBitsStr = version.split("[.\\-]");
+    return Version.fromBits(
+        Integer.parseInt(versionBitsStr[0]),
+        Integer.parseInt(versionBitsStr[1]),
+        Integer.parseInt(versionBitsStr[2]));
+  }
+
   // #12895: test on a carefully crafted 9.8.0 index (from a small contiguous subset
   // of wikibigall unique terms) that shows the read-time exception of
   // IntersectTermsEnum (used by WildcardQuery)

lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java

@@ -864,7 +864,7 @@ public void testFailOpenOldIndex() throws IOException {
     assertTrue(
         ex.getMessage()
             .contains(
-                "This Lucene version only supports indexes created with major version "
+                "This Lucene version only supports indexes with major version "
                     + Version.LATEST.major
                     + " or later"));
     // now open with allowed min version

lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java

@@ -328,7 +328,7 @@ public static final SegmentInfos readCommit(
         throw new IndexFormatTooOldException(
             input, magic, CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC);
       }
-      format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_74, VERSION_CURRENT);
+      format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_86, VERSION_CURRENT);
       byte[] id = new byte[StringHelper.ID_LENGTH];
       input.readBytes(id, 0, id.length);
       CodecUtil.checkIndexHeaderSuffix(input, Long.toString(generation, Character.MAX_RADIX));
@@ -346,30 +346,12 @@ public static final SegmentInfos readCommit(
             input);
       }
 
-      if (indexCreatedVersion < minSupportedMajorVersion) {
-        throw new IndexFormatTooOldException(
-            input,
-            "Index created with Lucene "
-                + indexCreatedVersion
-                + ".x is not supported by Lucene "
-                + Version.LATEST
-                + ". This Lucene version only supports indexes created with major version "
-                + minSupportedMajorVersion
-                + " or later (found: "
-                + indexCreatedVersion
-                + ", minimum: "
-                + minSupportedMajorVersion
-                + "). To resolve this issue: (1) Re-index your data using Lucene "
-                + Version.LATEST.major
-                + ".x, or (2) Use an older Lucene version that supports your index format.");
-      }
-
       SegmentInfos infos = new SegmentInfos(indexCreatedVersion);
       infos.id = id;
       infos.generation = generation;
       infos.lastGeneration = generation;
       infos.luceneVersion = luceneVersion;
-      parseSegmentInfos(directory, input, infos, format);
+      parseSegmentInfos(directory, input, infos, format, minSupportedMajorVersion);
       return infos;
 
     } catch (Throwable t) {
@@ -385,7 +367,12 @@ public static final SegmentInfos readCommit(
   }
 
   private static void parseSegmentInfos(
-      Directory directory, DataInput input, SegmentInfos infos, int format) throws IOException {
+      Directory directory,
+      DataInput input,
+      SegmentInfos infos,
+      int format,
+      int minSupportedMajorVersion)
+      throws IOException {
     infos.version = CodecUtil.readBELong(input);
     // System.out.println("READ sis version=" + infos.version);
     infos.counter = input.readVLong();
@@ -402,6 +389,7 @@ private static void parseSegmentInfos(
     }
 
     long totalDocs = 0;
+
     for (int seg = 0; seg < numSegments; seg++) {
       String segName = input.readString();
       byte[] segmentID = new byte[StringHelper.ID_LENGTH];
@@ -495,6 +483,30 @@ private static void parseSegmentInfos(
                 + infos.indexCreatedVersionMajor,
             input);
       }
+
+      int createdOrSegmentMinVersion =
+          info.getMinVersion() == null
+              ? infos.indexCreatedVersionMajor
+              : info.getMinVersion().major;
+
+      // version >=7 are expected to record minVersion
+      if (info.getMinVersion() == null || info.getMinVersion().major < minSupportedMajorVersion) {
+        throw new IndexFormatTooOldException(
+            input,
+            "Index has segments derived from Lucene version "
+                + createdOrSegmentMinVersion
+                + ".x and is not supported by Lucene "
+                + Version.LATEST
+                + ". This Lucene version only supports indexes with major version "
+                + minSupportedMajorVersion
+                + " or later (found: "
+                + createdOrSegmentMinVersion
+                + ", minimum supported: "
+                + minSupportedMajorVersion
+                + "). To resolve this issue re-index your data using Lucene "
+                + minSupportedMajorVersion
+                + ".x or later.");
+      }
     }
 
     infos.userData = input.readMapOfStrings();