SOLR-17725: Add MergePolicy to block older segments from participating in merges (#3883)

Helps with Solr upgrades in conjunction with re-indexing.
New: LatestVersionFilterMergePolicy

---------

Co-authored-by: Rahul Goswami <[email protected]>

Author: 2 people

changelog/unreleased/SOLR-17725-Merge-policy-to-upgrade-index-for-compatibility-with-future-Solr-version.yml

@@ -0,0 +1,8 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Merge policy to upgrade index for compatibility with future Solr version
+type: added
+authors:
+  - name: Rahul Goswami
+links:
+  - name: SOLR-17725
+    url: https://issues.apache.org/jira/browse/SOLR-17725

solr/core/src/java/org/apache/solr/index/LatestVersionFilterMergePolicy.java

@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.index;
+
+import java.io.IOException;
+import java.util.Map;
+import org.apache.lucene.index.FilterMergePolicy;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.MergeTrigger;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.util.Version;
+
+/**
+ * Prevents any older version segment (i.e. older than current lucene major version), either
+ * original or one derived as a result of merging with an older version segment, from being
+ * considered for merges. That way a snapshot of older segments remains consistent. This assists in
+ * upgrading to a future Lucene major version if existing documents are reindexed in the current
+ * version with this merge policy in place.
+ */
+public class LatestVersionFilterMergePolicy extends FilterMergePolicy {
+
+  public LatestVersionFilterMergePolicy(MergePolicy in) {
+    super(in);
+  }
+
+  @Override
+  public MergeSpecification findMerges(
+      MergeTrigger mergeTrigger, SegmentInfos infos, MergeContext mergeContext) throws IOException {
+    return in.findMerges(mergeTrigger, getFilteredInfos(infos), mergeContext);
+  }
+
+  @Override
+  public MergeSpecification findForcedMerges(
+      SegmentInfos infos,
+      int maxSegmentCount,
+      Map<SegmentCommitInfo, Boolean> segmentsToMerge,
+      MergeContext mergeContext)
+      throws IOException {
+    return in.findForcedMerges(
+        getFilteredInfos(infos), maxSegmentCount, segmentsToMerge, mergeContext);
+  }
+
+  @Override
+  public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext)
+      throws IOException {
+    return in.findForcedDeletesMerges(getFilteredInfos(infos), mergeContext);
+  }
+
+  @Override
+  public MergeSpecification findFullFlushMerges(
+      MergeTrigger mergeTrigger, SegmentInfos infos, MergeContext mergeContext) throws IOException {
+    return in.findFullFlushMerges(mergeTrigger, getFilteredInfos(infos), mergeContext);
+  }
+
+  private SegmentInfos getFilteredInfos(SegmentInfos infos) {
+    SegmentInfos infosClone = null;
+
+    for (SegmentCommitInfo info : infos) {
+      if (!allowSegmentForMerge(info)) {
+        // There are older version segments present.
+        // We should not remove from the original SegmentInfos. Hence we clone.
+        infosClone = infos.clone();
+        infosClone.clear();
+        break;
+      }
+    }
+
+    if (infosClone == null) {
+      // All segments are latest major version and allowed to participate in merge
+      return infos;
+    } else {
+      // Either mixed versions or all older version segments.
+      // If we are here, most runs should fall in the former case.
+      // The latter case should only happen once right after an upgrade, so we are ok with incurring
+      // this redundant iteration for that one time to keep the logic simple
+      for (SegmentCommitInfo info : infos) {
+        if (allowSegmentForMerge(info)) {
+          infosClone.add(info);
+        }
+      }
+    }
+
+    return infosClone;
+  }
+
+  /**
+   * Determines if a SegmentCommitInfo should be part of the candidate set of segments that will be
+   * considered for merges. By default, we only allow LATEST version segments to participate in
+   * merges.
+   */
+  protected boolean allowSegmentForMerge(SegmentCommitInfo info) {
+    return info.info.getMinVersion() != null
+        && info.info.getMinVersion().major == Version.LATEST.major;
+  }
+}

solr/core/src/java/org/apache/solr/index/LatestVersionMergePolicyFactory.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.index;
+
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.schema.IndexSchema;
+
+/**
+ * A {@link MergePolicyFactory} for {@link LatestVersionFilterMergePolicy} objects. The returned
+ * LatestVersionFilterMergePolicy instance blocks older version segments (< current version of
+ * Lucene) from participating in merges and delegates the merging to a TieredMergePolicy instance by
+ * default. This can be used to reindex the data and ensure all segments are the latest version
+ * segments by the end of the reindexing. This can help prepare the index for upgrade to a later
+ * version of Solr/Lucene even if it was initially created on a now unsupported version
+ */
+public class LatestVersionMergePolicyFactory extends SimpleMergePolicyFactory {
+
+  public LatestVersionMergePolicyFactory(
+      SolrResourceLoader resourceLoader, MergePolicyFactoryArgs args, IndexSchema schema) {
+    super(resourceLoader, args, schema);
+  }
+
+  @Override
+  protected MergePolicy getMergePolicyInstance() {
+    return new LatestVersionFilterMergePolicy(new TieredMergePolicy());
+  }
+}