DRILL-3742: Classpath scanning and build improvement

Makes the classpath scanning a build time class discovery
Makes the fmpp generation incremental
Removes some slowness in DrillBit closing
Reduces the build time by 30%

This closes apache#148

common/pom.xml

@@ -110,6 +110,25 @@
 
   <build>
     <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>1.2.1</version>
+        <executions>
+          <execution>
+            <phase>process-classes</phase>
+            <goals>
+              <goal>java</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <mainClass>org.apache.drill.common.scanner.BuildTimeScan</mainClass>
+          <arguments>
+            <argument>${project.build.outputDirectory}</argument>
+          </arguments>
+        </configuration>
+      </plugin>
       <plugin>
         <artifactId>maven-surefire-plugin</artifactId>
         <version>2.15</version>

common/src/main/java/org/apache/drill/common/JSONOptions.java

@@ -24,6 +24,7 @@
 import org.apache.drill.common.JSONOptions.De;
 import org.apache.drill.common.JSONOptions.Se;
 import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.config.LogicalPlanPersistence;
 import org.apache.drill.common.exceptions.LogicalPlanParsingException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -66,7 +67,7 @@ public JSONOptions(JsonNode n, JsonLocation location) {
   }
 
   @SuppressWarnings("unchecked")
-  public <T> T getWith(DrillConfig config, Class<T> c) {
+  public <T> T getWith(LogicalPlanPersistence lpPersistance, Class<T> c) {
     try {
       if (opaque != null) {
         final Class<?> opaqueClass = opaque.getClass();
@@ -88,15 +89,15 @@ public <T> T getWith(DrillConfig config, Class<T> c) {
       }
 
       //logger.debug("Read tree {}", root);
-      return config.getMapper().treeToValue(root, c);
+      return lpPersistance.getMapper().treeToValue(root, c);
     } catch (JsonProcessingException e) {
       throw new LogicalPlanParsingException(String.format("Failure while trying to convert late bound " +
         "json options to type of %s. Reference was originally located at line %d, column %d.",
         c.getCanonicalName(), location.getLineNr(), location.getColumnNr()), e);
     }
   }
 
-  public <T> T getListWith(DrillConfig config, TypeReference<T> t) throws IOException {
+  public <T> T getListWith(LogicalPlanPersistence config, TypeReference<T> t) throws IOException {
     return getListWith(config.getMapper(), t);
   }
 

common/src/main/java/org/apache/drill/common/config/CommonConstants.java

@@ -28,20 +28,4 @@ public interface CommonConstants {
   /** Override configuration file name.  (Classpath resource pathname.) */
   String CONFIG_OVERRIDE_RESOURCE_PATHNAME = "drill-override.conf";
 
-  /** Configuration pathname to list of names of packages to scan for logical
-   *  operator subclasses. */
-  String LOGICAL_OPERATOR_SCAN_PACKAGES = "drill.logical.operator.packages";
-
-  /** Configuration pathname to list of names of packages to scan for physical
-   *  operator subclasses. */
-  String PHYSICAL_OPERATOR_SCAN_PACKAGES = "drill.physical.operator.packages";
-
-  /** Configuration pathname to list of names of packages to scan for function
-   *  subclasses. */
-  String LOGICAL_FUNCTION_SCAN_PACKAGES = "drill.logical.function.packages";
-
-  /** Configuration pathname to list of packages to scan for storage plugin
-   *  configuration subclasses. */
-  String STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES = "drill.logical.storage.packages";
-
 }

common/src/main/java/org/apache/drill/common/config/DrillConfig.java

@@ -22,24 +22,16 @@
 import java.net.URL;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map.Entry;
 import java.util.Properties;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.drill.common.exceptions.DrillConfigurationException;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.FormatPluginConfigBase;
-import org.apache.drill.common.logical.StoragePluginConfigBase;
-import org.apache.drill.common.logical.data.LogicalOperatorBase;
-import org.apache.drill.common.util.PathScanner;
+import org.apache.drill.common.scanner.ClassPathScanner;
 import org.reflections.util.ClasspathHelper;
 
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser.Feature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableList;
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
@@ -48,7 +40,6 @@
 public final class DrillConfig extends NestedConfig{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConfig.class);
 
-  private final ObjectMapper mapper;
   private final ImmutableList<String> startupArguments;
 
   public static final boolean ON_OSX = System.getProperty("os.name").contains("OS X");
@@ -62,27 +53,10 @@ public DrillConfig(Config config, boolean enableServerConfigs) {
     logger.debug("Setting up DrillConfig object.");
     logger.trace("Given Config object is:\n{}",
                  config.root().render(ConfigRenderOptions.defaults()));
-    mapper = new ObjectMapper();
-
-    if (enableServerConfigs) {
-      SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule")
-        .addDeserializer(LogicalExpression.class, new LogicalExpression.De(this))
-        .addDeserializer(SchemaPath.class, new SchemaPath.De());
-
-      mapper.registerModule(deserModule);
-      mapper.enable(SerializationFeature.INDENT_OUTPUT);
-      mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
-      mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
-      mapper.configure(Feature.ALLOW_COMMENTS, true);
-      mapper.registerSubtypes(LogicalOperatorBase.getSubTypes(this));
-      mapper.registerSubtypes(StoragePluginConfigBase.getSubTypes(this));
-      mapper.registerSubtypes(FormatPluginConfigBase.getSubTypes(this));
-    }
-
     RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean();
     this.startupArguments = ImmutableList.copyOf(bean.getInputArguments());
     logger.debug("DrillConfig object initialized.");
-  };
+  }
 
   public List<String> getStartupArguments() {
     return startupArguments;
@@ -168,6 +142,8 @@ public static DrillConfig create(String overrideFileResourcePathname, boolean en
   private static DrillConfig create(String overrideFileResourcePathname,
                                     final Properties overriderProps,
                                     final boolean enableServerConfigs) {
+    final StringBuilder logString = new StringBuilder();
+    final Stopwatch watch = new Stopwatch().start();
     overrideFileResourcePathname =
         overrideFileResourcePathname == null
             ? CommonConstants.CONFIG_OVERRIDE_RESOURCE_PATHNAME
@@ -180,7 +156,7 @@ private static DrillConfig create(String overrideFileResourcePathname,
       final URL url =
           classLoader.getResource(CommonConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME);
       if (null != url) {
-        logger.info("Loading base configuration file at {}.", url);
+        logString.append("Base Configuration:\n\t- ").append(url).append("\n");
         fallback =
             ConfigFactory.load(classLoader,
                                CommonConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME);
@@ -189,14 +165,13 @@ private static DrillConfig create(String overrideFileResourcePathname,
     }
 
     // 2. Load per-module configuration files.
-    final Collection<URL> urls = PathScanner.getConfigURLs();
-    final String lineBrokenList =
-        urls.size() == 0 ? "" : "\n\t- " + Joiner.on("\n\t- ").join(urls);
-    logger.info("Loading {} module configuration files at: {}.",
-                urls.size(), lineBrokenList);
+    final Collection<URL> urls = ClassPathScanner.getConfigURLs();
+    logString.append("\nIntermediate Configuration and Plugin files, in order of precedence:\n");
     for (URL url : urls) {
+      logString.append("\t- ").append(url).append("\n");
       fallback = ConfigFactory.parseURL(url).withFallback(fallback);
     }
+    logString.append("\n");
 
     // 3. Load any specified overrides configuration file along with any
     //    overrides from JVM system properties (e.g., {-Dname=value").
@@ -205,19 +180,26 @@ private static DrillConfig create(String overrideFileResourcePathname,
     final URL overrideFileUrl =
         Thread.currentThread().getContextClassLoader().getResource(overrideFileResourcePathname);
     if (null != overrideFileUrl ) {
-      logger.info("Loading override config. file at {}.", overrideFileUrl);
+      logString.append("Override File: ").append(overrideFileUrl).append("\n");
     }
     Config effectiveConfig =
         ConfigFactory.load(overrideFileResourcePathname).withFallback(fallback);
 
     // 4. Apply any overriding properties.
     if (overriderProps != null) {
-      logger.info("Loading override Properties parameter {}.", overriderProps);
+      logString.append("Overridden Properties:\n");
+      for(Entry<Object, Object> entry : overriderProps.entrySet()){
+        logString.append("\t-").append(entry.getKey()).append(" = ").append(entry.getValue()).append("\n");
+      }
+      logString.append("\n");
       effectiveConfig =
           ConfigFactory.parseProperties(overriderProps).withFallback(effectiveConfig);
     }
 
     // 5. Create DrillConfig object from Config object.
+    logger.info("Configuration and plugin file(s) identified in {}ms.\n{}",
+        watch.elapsed(TimeUnit.MILLISECONDS),
+        logString);
     return new DrillConfig(effectiveConfig.resolve(), enableServerConfigs);
   }
 
@@ -256,10 +238,6 @@ public <T> T getInstanceOf(String location, Class<T> clazz) throws DrillConfigur
     }
   }
 
-  public ObjectMapper getMapper() {
-    return mapper;
-  }
-
   @Override
   public String toString() {
     return this.root().render();

common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java

@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.config;
+
+import java.util.Set;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.FormatPluginConfigBase;
+import org.apache.drill.common.logical.StoragePluginConfigBase;
+import org.apache.drill.common.logical.data.LogicalOperatorBase;
+import org.apache.drill.common.scanner.persistence.ScanResult;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser.Feature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+
+public class LogicalPlanPersistence {
+  private ObjectMapper mapper;
+
+  public ObjectMapper getMapper() {
+    return mapper;
+  }
+
+  public LogicalPlanPersistence(DrillConfig conf, ScanResult scanResult) {
+    mapper = new ObjectMapper();
+
+    SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule")
+        .addDeserializer(LogicalExpression.class, new LogicalExpression.De(conf))
+        .addDeserializer(SchemaPath.class, new SchemaPath.De());
+
+    mapper.registerModule(deserModule);
+    mapper.enable(SerializationFeature.INDENT_OUTPUT);
+    mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+    mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
+    mapper.configure(Feature.ALLOW_COMMENTS, true);
+    registerSubtypes(LogicalOperatorBase.getSubTypes(scanResult));
+    registerSubtypes(StoragePluginConfigBase.getSubTypes(scanResult));
+    registerSubtypes(FormatPluginConfigBase.getSubTypes(scanResult));
+  }
+
+  private <T> void registerSubtypes(Set<Class<? extends T>> types) {
+    for (Class<? extends T> type : types) {
+      mapper.registerSubtypes(type);
+    }
+  }
+}

common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java

@@ -17,35 +17,24 @@
  */
 package org.apache.drill.common.logical;
 
-import java.util.List;
+import java.util.Set;
 
-import org.apache.drill.common.config.CommonConstants;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.PathScanner;
-
-import com.google.common.base.Joiner;
+import org.apache.drill.common.scanner.persistence.ScanResult;
 
 
 public abstract class FormatPluginConfigBase implements FormatPluginConfig{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatPluginConfigBase.class);
 
 
   /**
-   * Use reflection to scan for implementations of {@see FormatPlugin}.
+   * scan for implementations of {@see FormatPlugin}.
    *
-   * @param config - Drill configuration object, used to find the packages to scan
+   * @param classpathScan - Drill configuration object, used to find the packages to scan
    * @return - list of classes that implement the interface.
    */
-  public synchronized static Class<?>[] getSubTypes(final DrillConfig config) {
-    final List<String> packages =
-        config.getStringList(CommonConstants.STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES);
-    final Class<?>[] pluginClasses =
-        PathScanner.scanForImplementationsArr(FormatPluginConfig.class, packages);
-    final String lineBrokenList =
-        pluginClasses.length == 0
-        ? "" : "\n\t- " + Joiner.on("\n\t- ").join(pluginClasses);
-    logger.debug("Found {} format plugin configuration classes: {}.",
-                 pluginClasses.length, lineBrokenList);
+  public static Set<Class<? extends FormatPluginConfig>> getSubTypes(final ScanResult classpathScan) {
+    final Set<Class<? extends FormatPluginConfig>> pluginClasses = classpathScan.getImplementations(FormatPluginConfig.class);
+    logger.debug("Found {} format plugin configuration classes: {}.", pluginClasses.size(), pluginClasses);
     return pluginClasses;
   }
 

common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 
 import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.config.LogicalPlanPersistence;
 import org.apache.drill.common.graph.Graph;
 import org.apache.drill.common.graph.GraphAlgos;
 import org.apache.drill.common.logical.data.LogicalOperator;
@@ -80,11 +81,11 @@ public Map<String, StoragePluginConfig> getStorageEngines() {
     return storageEngineMap;
   }
 
-  public String toJsonString(DrillConfig config) throws JsonProcessingException {
+  public String toJsonString(LogicalPlanPersistence config) throws JsonProcessingException {
     return config.getMapper().writeValueAsString(this);
   }
 
-  public String toJsonStringSafe(DrillConfig config){
+  public String toJsonStringSafe(LogicalPlanPersistence config){
     try{
       return toJsonString(config);
     }catch(JsonProcessingException e){
@@ -94,7 +95,7 @@ public String toJsonStringSafe(DrillConfig config){
   }
 
   /** Parses a logical plan. */
-  public static LogicalPlan parse(DrillConfig config, String planString) {
+  public static LogicalPlan parse(LogicalPlanPersistence config, String planString) {
     ObjectMapper mapper = config.getMapper();
     try {
       LogicalPlan plan = mapper.readValue(planString, LogicalPlan.class);
@@ -106,7 +107,7 @@ public static LogicalPlan parse(DrillConfig config, String planString) {
   }
 
   /** Converts a logical plan to a string. (Opposite of {@link #parse}.) */
-  public String unparse(DrillConfig config) {
+  public String unparse(LogicalPlanPersistence config) {
     try {
       return config.getMapper().writeValueAsString(this);
     } catch (JsonProcessingException e) {