Rewrite percolator queries in legacy format at query time

Rewriting the query at percolate time, because this is sometimes necessary:
* From 5.0 and onwards the percolator rewrites the query at index time,
   this is not the case for percolator queries in indices created before 5.0
   Doing so fixes percolator query upgrade issues.

Closes #24485

3 files changed, 29 insertions, 17 deletions

diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index 07677c25e1..e26c4ba9f9 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -36,6 +36,7 @@ import org.apache.lucene.search.TermInSetQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
@@ -284,7 +285,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         );
         verifyQuery(queryBuilder);
         // Fetching of terms, shapes and indexed scripts happen during this rewrite:
-        queryBuilder = queryBuilder.rewrite(queryShardContext);
+        queryBuilder = QueryBuilder.rewriteQuery(queryBuilder, queryShardContext);
 
         try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) {
             queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap()));
@@ -346,6 +347,14 @@ public class PercolatorFieldMapper extends FieldMapper {
         // as an analyzed string.
         context.setAllowUnmappedFields(false);
         context.setMapUnmappedFieldAsString(mapUnmappedFieldsAsString);
+
+        // Rewriting the query at percolate time, because this is sometimes necessary:
+        // * From 5.0 and onwards the percolator rewrites the query at index time,
+        //   this is not the case for percolator queries in indices created before 5.0
+        if (context.getIndexSettings().getIndexVersionCreated().before(Version.V_5_0_0_alpha1)) {
+            queryBuilder = QueryBuilder.rewriteQuery(queryBuilder, context);
+        }
+
         return queryBuilder.toQuery(context);
     }
 
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
index 48459858ba..80f8cb1f4d 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.percolator;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -44,7 +43,6 @@ import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.index.query.QueryBuilders.scriptQuery;
 import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
@@ -72,7 +70,7 @@ public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase {
 
         // verify cluster state:
         ClusterState state = client().admin().cluster().prepareState().get().getState();
-        assertThat(state.metaData().indices().size(), equalTo(1));
+        assertThat(state.metaData().indices().size(), equalTo(2));
         assertThat(state.metaData().indices().get(INDEX_NAME), notNullValue());
         assertThat(state.metaData().indices().get(INDEX_NAME).getCreationVersion(), equalTo(Version.V_2_0_0));
         assertThat(state.metaData().indices().get(INDEX_NAME).getUpgradedVersion(), equalTo(Version.CURRENT));
@@ -88,17 +86,18 @@ public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase {
             .setTypes(".percolator")
             .addSort("_uid", SortOrder.ASC)
             .get();
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(4L));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
-        assertThat(searchResponse.getHits().getAt(1).id(), equalTo("2"));
-        assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3"));
-        assertThat(searchResponse.getHits().getAt(3).id(), equalTo("4"));
+        assertThat(searchResponse.getHits().getTotalHits(), equalTo(5L));
+        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("0"));
+        assertThat(searchResponse.getHits().getAt(1).id(), equalTo("1"));
+        assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2"));
+        assertThat(searchResponse.getHits().getAt(3).id(), equalTo("3"));
+        assertThat(searchResponse.getHits().getAt(4).id(), equalTo("4"));
         assertThat(XContentMapValues.extractValue("query.script.script.inline",
-                searchResponse.getHits().getAt(3).sourceAsMap()), equalTo("return true"));
+                searchResponse.getHits().getAt(4).sourceAsMap()), equalTo("return true"));
         // we don't upgrade the script definitions so that they include explicitly the lang,
         // because we read / parse the query at search time.
         assertThat(XContentMapValues.extractValue("query.script.script.lang",
-                searchResponse.getHits().getAt(3).sourceAsMap()), nullValue());
+                searchResponse.getHits().getAt(4).sourceAsMap()), nullValue());
 
         // verify percolate response
         PercolateResponse percolateResponse = preparePercolate(client())
@@ -107,9 +106,10 @@ public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase {
                 .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
                 .get();
 
-        assertThat(percolateResponse.getCount(), equalTo(1L));
-        assertThat(percolateResponse.getMatches().length, equalTo(1));
+        assertThat(percolateResponse.getCount(), equalTo(2L));
+        assertThat(percolateResponse.getMatches().length, equalTo(2));
         assertThat(percolateResponse.getMatches()[0].getId().string(), equalTo("4"));
+        assertThat(percolateResponse.getMatches()[1].getId().string(), equalTo("0"));
 
         percolateResponse = preparePercolate(client())
             .setIndices(INDEX_NAME)
@@ -117,11 +117,12 @@ public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase {
             .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("message", "the quick brown fox jumps over the lazy dog"))
             .get();
 
-        assertThat(percolateResponse.getCount(), equalTo(3L));
-        assertThat(percolateResponse.getMatches().length, equalTo(3));
+        assertThat(percolateResponse.getCount(), equalTo(4L));
+        assertThat(percolateResponse.getMatches().length, equalTo(4));
         assertThat(percolateResponse.getMatches()[0].getId().string(), equalTo("1"));
         assertThat(percolateResponse.getMatches()[1].getId().string(), equalTo("2"));
         assertThat(percolateResponse.getMatches()[2].getId().string(), equalTo("4"));
+        assertThat(percolateResponse.getMatches()[3].getId().string(), equalTo("0"));
 
         // add an extra query and verify the results
         client().prepareIndex(INDEX_NAME, ".percolator", "5")
@@ -135,11 +136,13 @@ public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase {
             .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("message", "the quick brown fox jumps over the lazy dog"))
             .get();
 
-        assertThat(percolateResponse.getCount(), equalTo(4L));
-        assertThat(percolateResponse.getMatches().length, equalTo(4));
+        assertThat(percolateResponse.getCount(), equalTo(5L));
+        assertThat(percolateResponse.getMatches().length, equalTo(5));
         assertThat(percolateResponse.getMatches()[0].getId().string(), equalTo("1"));
         assertThat(percolateResponse.getMatches()[1].getId().string(), equalTo("2"));
         assertThat(percolateResponse.getMatches()[2].getId().string(), equalTo("4"));
+        assertThat(percolateResponse.getMatches()[3].getId().string(), equalTo("0"));
+        assertThat(percolateResponse.getMatches()[4].getId().string(), equalTo("5"));
     }
 
     private void setupNode() throws Exception {
diff --git a/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip b/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip
index 43a8cceb19..777597abc4 100644
--- a/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip
+++ b/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip