SOLR-4587: integrate lucene-monitor into solr

settings.gradle

@@ -52,6 +52,7 @@ include "solr:modules:ltr"
 include "solr:modules:s3-repository"
 include "solr:modules:scripting"
 include "solr:modules:sql"
+include "solr:modules:monitor"
 include "solr:webapp"
 include "solr:benchmark"
 include "solr:test-framework"

solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java

@@ -105,15 +105,7 @@ public void process(ResponseBuilder rb) throws IOException {
         info.addAll(stdinfo);
       }
 
-      FacetDebugInfo fdebug = (FacetDebugInfo) (rb.req.getContext().get("FacetDebugInfo"));
-      if (fdebug != null) {
-        info.add("facet-trace", fdebug.getFacetDebugInfo());
-      }
-
-      fdebug = (FacetDebugInfo) (rb.req.getContext().get("FacetDebugInfo-nonJson"));
-      if (fdebug != null) {
-        info.add("facet-debug", fdebug.getFacetDebugInfo());
-      }
+      addCustomInfo(rb, info);
 
       if (rb.req.getJSON() != null) {
         info.add(JSON, rb.req.getJSON());
@@ -139,6 +131,18 @@ public void process(ResponseBuilder rb) throws IOException {
     }
   }
 
+  protected void addCustomInfo(ResponseBuilder rb, NamedList<Object> info) {
+    FacetDebugInfo fdebug = (FacetDebugInfo) (rb.req.getContext().get("FacetDebugInfo"));
+    if (fdebug != null) {
+      info.add("facet-trace", fdebug.getFacetDebugInfo());
+    }
+
+    fdebug = (FacetDebugInfo) (rb.req.getContext().get("FacetDebugInfo-nonJson"));
+    if (fdebug != null) {
+      info.add("facet-debug", fdebug.getFacetDebugInfo());
+    }
+  }
+
   private void doDebugTrack(ResponseBuilder rb) {
     final String rid = rb.req.getParams().get(CommonParams.REQUEST_ID);
     rb.addDebug(rid, "track", CommonParams.REQUEST_ID); // to see it in the response

solr/licenses/lucene-monitor-9.11.1.jar.sha1

@@ -0,0 +1 @@
+fa731be343ec79d940c7abd0dd8ff1de9bf9c7f1

solr/modules/monitor/build.gradle

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: 'java-library'
+
+description = 'Apache Solr Monitor'
+
+dependencies {
+
+    implementation project(":solr:core")
+    implementation project(":solr:solrj")
+    implementation "org.apache.lucene:lucene-core"
+    implementation "org.apache.lucene:lucene-monitor"
+    implementation 'com.github.ben-manes.caffeine:caffeine'
+    testImplementation project(':solr:test-framework')
+    testImplementation 'junit:junit'
+}
+
+

solr/modules/monitor/src/java/org/apache/lucene/monitor/CandidateMatcher.java

@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.monitor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+
+/** Class used to match candidate queries selected by a Presearcher from a Monitor query index. */
+public abstract class CandidateMatcher<T extends QueryMatch> {
+
+  /** The searcher to run candidate queries against */
+  protected final IndexSearcher searcher;
+
+  private final Map<String, Exception> errors = new HashMap<>();
+  private final List<MatchHolder<T>> matches;
+
+  private long searchTime = System.nanoTime();
+
+  private static class MatchHolder<T> {
+    Map<String, T> matches = new HashMap<>();
+  }
+
+  /**
+   * Creates a new CandidateMatcher for the supplied DocumentBatch
+   *
+   * @param searcher the IndexSearcher to run queries against
+   */
+  public CandidateMatcher(IndexSearcher searcher) {
+    this.searcher = searcher;
+    int docCount = searcher.getIndexReader().maxDoc();
+    this.matches = new ArrayList<>(docCount);
+    for (int i = 0; i < docCount; i++) {
+      this.matches.add(new MatchHolder<>());
+    }
+  }
+
+  /**
+   * Runs the supplied query against this CandidateMatcher's set of documents, storing any resulting
+   * match, and recording the query in the presearcher hits
+   *
+   * @param queryId the query id
+   * @param matchQuery the query to run
+   * @param metadata the query metadata
+   * @throws IOException on IO errors
+   */
+  public abstract void matchQuery(String queryId, Query matchQuery, Map<String, String> metadata)
+      throws IOException;
+
+  /**
+   * Record a match
+   *
+   * @param match a QueryMatch object
+   */
+  protected final void addMatch(T match, int doc) {
+    MatchHolder<T> docMatches = matches.get(doc);
+    docMatches.matches.compute(
+        match.getQueryId(),
+        (key, oldValue) -> {
+          if (oldValue != null) {
+            return resolve(match, oldValue);
+          }
+          return match;
+        });
+  }
+
+  /**
+   * If two matches from the same query are found (for example, two branches of a disjunction),
+   * combine them.
+   *
+   * @param match1 the first match found
+   * @param match2 the second match found
+   * @return a Match object that combines the two
+   */
+  public abstract T resolve(T match1, T match2);
+
+  /** Called by the Monitor if running a query throws an Exception */
+  void reportError(String queryId, Exception e) {
+    this.errors.put(queryId, e);
+  }
+
+  /**
+   * @return the matches from this matcher
+   */
+  public final MultiMatchingQueries<T> finish(long buildTime, int queryCount) {
+    doFinish();
+    this.searchTime =
+        TimeUnit.MILLISECONDS.convert(System.nanoTime() - searchTime, TimeUnit.NANOSECONDS);
+    List<Map<String, T>> results = new ArrayList<>();
+    for (MatchHolder<T> matchHolder : matches) {
+      results.add(matchHolder.matches);
+    }
+    return new MultiMatchingQueries<>(
+        results, errors, buildTime, searchTime, queryCount, matches.size());
+  }
+
+  /** Called when all monitoring of a batch of documents is complete */
+  protected void doFinish() {}
+
+  /** Copy all matches from another CandidateMatcher */
+  protected void copyMatches(CandidateMatcher<T> other) {
+    this.matches.clear();
+    this.matches.addAll(other.matches);
+  }
+}

solr/modules/monitor/src/java/org/apache/lucene/monitor/DocumentBatchVisitor.java

@@ -0,0 +1,64 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  * contributor license agreements.  See the NOTICE file distributed with
+ *  * this work for additional information regarding copyright ownership.
+ *  * The ASF licenses this file to You under the Apache License, Version 2.0
+ *  * (the "License"); you may not use this file except in compliance with
+ *  * the License.  You may obtain a copy of the License at
+ *  *
+ *  *     http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.apache.lucene.monitor;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.LeafReader;
+
+public class DocumentBatchVisitor implements Closeable, Supplier<LeafReader> {
+
+  private final DocumentBatch batch;
+  private final List<Document> docs;
+
+  private DocumentBatchVisitor(DocumentBatch batch, List<Document> docs) {
+    this.batch = batch;
+    this.docs = docs;
+  }
+
+  public static DocumentBatchVisitor of(Analyzer analyzer, List<Document> docs) {
+    return new DocumentBatchVisitor(
+        DocumentBatch.of(analyzer, docs.toArray(new Document[0])), docs);
+  }
+
+  @Override
+  public void close() throws IOException {
+    batch.close();
+  }
+
+  @Override
+  public LeafReader get() {
+    return batch.get();
+  }
+
+  public int size() {
+    return docs.size();
+  }
+
+  @Override
+  public String toString() {
+    return docs.stream().map(Document::toString).collect(Collectors.joining(" "));
+  }
+}

solr/modules/monitor/src/java/org/apache/lucene/monitor/MatchesAggregator.java

@@ -0,0 +1,60 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  * contributor license agreements.  See the NOTICE file distributed with
+ *  * this work for additional information regarding copyright ownership.
+ *  * The ASF licenses this file to You under the Apache License, Version 2.0
+ *  * (the "License"); you may not use this file except in compliance with
+ *  * the License.  You may obtain a copy of the License at
+ *  *
+ *  *     http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+package org.apache.lucene.monitor;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.lucene.search.Query;
+
+public class MatchesAggregator<T extends QueryMatch> extends CandidateMatcher<T> {
+
+  private final CandidateMatcher<T> resolvingMatcher;
+
+  private MatchesAggregator(
+      List<CandidateMatcher<T>> matchers, CandidateMatcher<T> resolvingMatcher) {
+    super(resolvingMatcher.searcher);
+    this.resolvingMatcher = resolvingMatcher;
+    for (var matcher : matchers) {
+      var matches = matcher.finish(Long.MIN_VALUE, -1);
+      for (int doc = 0; doc < matches.getBatchSize(); doc++) {
+        for (T match : matches.getMatches(doc)) {
+          this.addMatch(match, doc);
+        }
+      }
+      for (Map.Entry<String, Exception> error : matches.getErrors().entrySet()) {
+        this.reportError(error.getKey(), error.getValue());
+      }
+    }
+  }
+
+  @Override
+  public void matchQuery(String queryId, Query matchQuery, Map<String, String> metadata) {
+    throw new UnsupportedOperationException("only use for aggregating other matchers");
+  }
+
+  @Override
+  public T resolve(T match1, T match2) {
+    return resolvingMatcher.resolve(match1, match2);
+  }
+
+  public static <T extends QueryMatch> MultiMatchingQueries<T> aggregate(
+      List<CandidateMatcher<T>> matchers, CandidateMatcher<T> resolver, int queryCount) {
+    return new MatchesAggregator<>(matchers, resolver).finish(Long.MIN_VALUE, queryCount);
+  }
+}

solr/modules/monitor/src/java/org/apache/lucene/monitor/MonitorFields.java

@@ -0,0 +1,32 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  * contributor license agreements.  See the NOTICE file distributed with
+ *  * this work for additional information regarding copyright ownership.
+ *  * The ASF licenses this file to You under the Apache License, Version 2.0
+ *  * (the "License"); you may not use this file except in compliance with
+ *  * the License.  You may obtain a copy of the License at
+ *  *
+ *  *     http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.apache.lucene.monitor;
+
+import java.util.Set;
+
+public class MonitorFields {
+
+  public static final String QUERY_ID = QueryIndex.FIELDS.query_id + "_";
+  public static final String CACHE_ID = QueryIndex.FIELDS.cache_id + "_";
+  public static final String MONITOR_QUERY = QueryIndex.FIELDS.mq + "_";
+
+  public static final Set<String> REQUIRED_MONITOR_SCHEMA_FIELDS =
+      Set.of(QUERY_ID, CACHE_ID, MONITOR_QUERY);
+}