DRILL-4877: If pruning was not applicable only keep the selectionRoot…

… in the entries field.
adityakishore · Sep 8, 2016 · 18866d5 · 18866d5
1 parent f11f321
commit 18866d5
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 11 deletions.
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MetadataContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MetadataContext.java
@@ -67,12 +67,8 @@ public void setPruneStatus(PruneStatus status) {
     pruneStatus = status;
   }
 
-  public boolean wasPruningStarted() {
-    return pruneStatus != PruneStatus.NOT_STARTED;
-  }
-
-  public boolean wasPruned() {
-    return pruneStatus == PruneStatus.PRUNED;
+  public PruneStatus getPruneStatus() {
+    return pruneStatus;
   }
 
 }

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -49,6 +49,7 @@
 import org.apache.drill.exec.store.dfs.DrillPathFilter;
 import org.apache.drill.exec.store.dfs.FileSelection;
 import org.apache.drill.exec.store.dfs.MetadataContext;
+import org.apache.drill.exec.store.dfs.MetadataContext.PruneStatus;
 import org.apache.drill.exec.store.dfs.ReadEntryFromHDFS;
 import org.apache.drill.exec.store.dfs.ReadEntryWithPath;
 import org.apache.drill.exec.store.dfs.easy.FileWork;
@@ -174,11 +175,11 @@ public ParquetGroupScan( //
 
     this.entries = Lists.newArrayList();
     if (fileSelection.getMetaContext() != null &&
-        (fileSelection.getMetaContext().wasPruningStarted() &&
-        ! fileSelection.getMetaContext().wasPruned())) {
-      // if pruning was attempted and nothing was pruned, initialize the entries with just
-      // the selection root instead of the fully expanded list to reduce overhead. The fully
-      // expanded list is already stored as part of the fileSet.
+        (fileSelection.getMetaContext().getPruneStatus() == PruneStatus.NOT_STARTED ||
+          fileSelection.getMetaContext().getPruneStatus() == PruneStatus.NOT_PRUNED)) {
+      // if pruning was not applicable or was attempted and nothing was pruned, initialize the
+      // entries with just the selection root instead of the fully expanded list to reduce overhead.
+      // The fully expanded list is already stored as part of the fileSet.
       // TODO: at some point we should examine whether the list of entries is absolutely needed.
       entries.add(new ReadEntryWithPath(fileSelection.getSelectionRoot()));
     } else {

diff --git a/...java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java b/...java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
@@ -371,7 +371,32 @@ public void testDrill4786_2() throws Exception {
 
   }
 
+  @Test // DRILL-4877
+  public void testDrill4877() throws Exception {
+    // create metadata cache
+    test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2));
+    checkForMetadataFile(tableName2);
+
+    // run query and check correctness
+    String query1 = String.format("select max(dir0) as max0, max(dir1) as max1 from dfs_test.`%s/%s` ",
+        getDfsTestTmpSchemaLocation(), tableName2);
+
+    testBuilder()
+    .sqlQuery(query1)
+      .unOrdered()
+      .baselineColumns("max0", "max1")
+      .baselineValues("1995", "Q4")
+      .go();
+
+    int expectedNumFiles = 1; // point to selectionRoot since no pruning is done in this query
 
+    String numFilesPattern = "numFiles=" + expectedNumFiles;
+    String usedMetaPattern = "usedMetadataFile=true";
+    String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tableName2);
+    PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
+        new String[] {});
+
+  }
 
   private void checkForMetadataFile(String table) throws Exception {
     String tmpDir = getDfsTestTmpSchemaLocation();