Ignore date ranges containing 'now' when pre-processing a percolator query (elastic#35160)

Today when a percolator query contains a date range then the query
analyzer extracts that range, so that at search time the `percolate` query
can exclude percolator queries efficiently that are never going to match.

The problem is that if 'now' is used it is evaluated at index time.
So the idea is to rewrite date ranges with 'now' to a match all query,
so that the query analyzer can't extract it and the `percolate` query
is  then able to evaluate 'now' at query time.

Author: martijnvg

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java

@@ -423,7 +423,15 @@ public void parse(ParseContext context) throws IOException {
 
         Version indexVersion = context.mapperService().getIndexSettings().getIndexVersionCreated();
         createQueryBuilderField(indexVersion, queryBuilderField, queryBuilder, context);
-        Query query = toQuery(queryShardContext, isMapUnmappedFieldAsText(), queryBuilder);
+
+        QueryBuilder queryBuilderForProcessing = queryBuilder.rewrite(new QueryShardContext(queryShardContext) {
+
+            @Override
+            public boolean convertNowRangeToMatchAll() {
+                return true;
+            }
+        });
+        Query query = toQuery(queryShardContext, isMapUnmappedFieldAsText(), queryBuilderForProcessing);
         processQuery(query, context);
     }
 

modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java

@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.percolator;
 
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
@@ -28,10 +30,14 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.query.Operator;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
@@ -49,9 +55,14 @@
 import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
+import static org.elasticsearch.index.query.QueryBuilders.scriptQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.hamcrest.Matchers.equalTo;
 
 public class PercolatorQuerySearchTests extends ESSingleNodeTestCase {
 
@@ -241,4 +252,53 @@ public void testMapUnmappedFieldAsString() throws IOException {
         assertSettingDeprecationsAndWarnings(new Setting[]{PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING});
     }
 
+    public void testRangeQueriesWithNow() throws Exception {
+        IndexService indexService = createIndex("test", Settings.builder().put("index.number_of_shards", 1).build(), "_doc",
+            "field1", "type=keyword", "field2", "type=date", "query", "type=percolator");
+
+        client().prepareIndex("test", "_doc", "1")
+            .setSource(jsonBuilder().startObject().field("query", rangeQuery("field2").from("now-1h").to("now+1h")).endObject())
+            .get();
+        client().prepareIndex("test", "_doc", "2")
+            .setSource(jsonBuilder().startObject().field("query", boolQuery()
+                .filter(termQuery("field1", "value"))
+                .filter(rangeQuery("field2").from("now-1h").to("now+1h"))
+            ).endObject())
+            .get();
+
+
+        Script script = new Script(ScriptType.INLINE, MockScriptPlugin.NAME, "1==1", Collections.emptyMap());
+        client().prepareIndex("test", "_doc", "3")
+            .setSource(jsonBuilder().startObject().field("query", boolQuery()
+                .filter(scriptQuery(script))
+                .filter(rangeQuery("field2").from("now-1h").to("now+1h"))
+            ).endObject())
+            .get();
+        client().admin().indices().prepareRefresh().get();
+
+        try (Engine.Searcher engineSearcher = indexService.getShard(0).acquireSearcher("test")) {
+            IndexSearcher indexSearcher = engineSearcher.searcher();
+            long[] currentTime = new long[] {System.currentTimeMillis()};
+            QueryShardContext queryShardContext =
+                indexService.newQueryShardContext(0, engineSearcher.reader(), () -> currentTime[0], null);
+
+            BytesReference source = BytesReference.bytes(jsonBuilder().startObject()
+                .field("field1", "value")
+                .field("field2", currentTime[0])
+                .endObject());
+            QueryBuilder queryBuilder = new PercolateQueryBuilder("query", source, XContentType.JSON);
+            Query query = queryBuilder.toQuery(queryShardContext);
+            assertThat(indexSearcher.count(query), equalTo(3));
+
+            currentTime[0] = currentTime[0] + 10800000; // + 3 hours
+            source = BytesReference.bytes(jsonBuilder().startObject()
+                .field("field1", "value")
+                .field("field2", currentTime[0])
+                .endObject());
+            queryBuilder = new PercolateQueryBuilder("query", source, XContentType.JSON);
+            query = queryBuilder.toQuery(queryShardContext);
+            assertThat(indexSearcher.count(query), equalTo(3));
+        }
+    }
+
 }

server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java

@@ -124,4 +124,15 @@ public void onFailure(Exception e) {
         }
     }
 
+    /**
+     * In pre-processing contexts that happen at index time 'now' date ranges should be replaced by a {@link MatchAllQueryBuilder}.
+     * Otherwise documents that should match at query time would never match and the document that have fallen outside the
+     * date range would continue to match.
+     *
+     * @return indicates whether range queries with date ranges using 'now' are rewritten to a {@link MatchAllQueryBuilder}.
+     */
+    public boolean convertNowRangeToMatchAll() {
+        return false;
+    }
+
 }

server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java

@@ -464,6 +464,16 @@ protected MappedFieldType.Relation getRelation(QueryRewriteContext queryRewriteC
 
     @Override
     protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        // Percolator queries get rewritten and pre-processed at index time.
+        // If a range query has a date range using 'now' and 'now' gets resolved at index time then
+        // the pre-processing uses that to pre-process. This can then lead to mismatches at query time.
+        if (queryRewriteContext.convertNowRangeToMatchAll()) {
+            if ((from() != null && from().toString().contains("now")) ||
+                (to() != null && to().toString().contains("now"))) {
+                return new MatchAllQueryBuilder();
+            }
+        }
+
         final MappedFieldType.Relation relation = getRelation(queryRewriteContext);
         switch (relation) {
         case DISJOINT:

server/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java

@@ -575,4 +575,33 @@ public void testParseRelation() {
         builder.relation("intersects");
         assertEquals(ShapeRelation.INTERSECTS, builder.relation());
     }
+
+    public void testConvertNowRangeToMatchAll() throws IOException {
+        RangeQueryBuilder query = new RangeQueryBuilder(DATE_FIELD_NAME);
+        DateTime queryFromValue = new DateTime(2019, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        DateTime queryToValue = new DateTime(2020, 1, 1, 0, 0, 0, ISOChronology.getInstanceUTC());
+        if (randomBoolean()) {
+            query.from("now");
+            query.to(queryToValue);
+        } else if (randomBoolean()) {
+            query.from(queryFromValue);
+            query.to("now");
+        } else {
+            query.from("now");
+            query.to("now+1h");
+        }
+        QueryShardContext queryShardContext = createShardContext();
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(RangeQueryBuilder.class));
+
+        queryShardContext = new QueryShardContext(queryShardContext) {
+
+            @Override
+            public boolean convertNowRangeToMatchAll() {
+                return true;
+            }
+        };
+        rewritten = query.rewrite(queryShardContext);
+        assertThat(rewritten, instanceOf(MatchAllQueryBuilder.class));
+    }
 }