Convert pair returned by first/last aggregators to number (#5566)

.travis.yml

@@ -47,6 +47,7 @@ matrix:
     - sudo: false
       env:
         - NAME="other modules test"
+        - AWS_REGION=us-east-1 # set a aws region for unit tests
       install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B
       before_script:
         - unset _JAVA_OPTIONS

api/src/main/java/io/druid/data/input/Rows.java

@@ -23,6 +23,7 @@
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Maps;
 import com.google.common.primitives.Longs;
+import io.druid.java.util.common.Pair;
 import io.druid.java.util.common.StringUtils;
 import io.druid.java.util.common.parsers.ParseException;
 
@@ -116,6 +117,8 @@ public static Number objectToNumber(final String name, final Object inputValue)
       catch (Exception e) {
         throw new ParseException(e, "Unable to parse value[%s] for field[%s]", inputValue, name);
       }
+    } else if (inputValue instanceof Pair) {
+      return objectToNumber(name, ((Pair) inputValue).rhs);
     } else {
       throw new ParseException("Unknown type[%s] for field", inputValue.getClass(), inputValue);
     }

api/src/main/java/io/druid/data/input/impl/MapInputRowParser.java

@@ -72,7 +72,7 @@ public List<InputRow> parseBatch(Map<String, Object> theMap)
       }
     }
     catch (Exception e) {
-      throw new ParseException(e, "Unparseable timestamp found!");
+      throw new ParseException(e, "Unparseable timestamp found! Event: %s", theMap);
     }
 
     return ImmutableList.of(new MapBasedInputRow(timestamp.getMillis(), dimensions, theMap));

api/src/main/java/io/druid/data/input/impl/StringInputRowParser.java

@@ -21,7 +21,6 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterators;
 import io.druid.data.input.ByteBufferInputRowParser;
@@ -36,14 +35,15 @@
 import java.nio.charset.Charset;
 import java.nio.charset.CoderResult;
 import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
 
 /**
  */
 public class StringInputRowParser implements ByteBufferInputRowParser
 {
-  private static final Charset DEFAULT_CHARSET = Charsets.UTF_8;
+  private static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
 
   private final ParseSpec parseSpec;
   private final MapInputRowParser mapParser;

api/src/main/java/io/druid/guice/JsonConfigProvider.java

@@ -130,8 +130,8 @@ public static <T> void bind(
       Key<Supplier<T>> supplierKey
   )
   {
-    binder.bind(supplierKey).toProvider((Provider) of(propertyBase, clazz)).in(LazySingleton.class);
-    binder.bind(instanceKey).toProvider(new SupplierProvider<T>(supplierKey));
+    binder.bind(supplierKey).toProvider(of(propertyBase, clazz)).in(LazySingleton.class);
+    binder.bind(instanceKey).toProvider(new SupplierProvider<>(supplierKey));
   }
 
   @SuppressWarnings("unchecked")

api/src/main/java/io/druid/indexer/IngestionState.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.indexer;
+
+public enum IngestionState
+{
+  NOT_STARTED,
+  DETERMINE_PARTITIONS,
+  BUILD_SEGMENTS,
+  COMPLETED
+}

api/src/main/java/io/druid/indexer/TaskMetricsGetter.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.indexer;
+
+import java.util.List;
+import java.util.Map;
+
+public interface TaskMetricsGetter
+{
+  List<String> getKeys();
+  Map<String, Number> getTotalMetrics();
+}

api/src/main/java/io/druid/indexer/TaskMetricsUtils.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.indexer;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+public class TaskMetricsUtils
+{
+  public static final String ROWS_PROCESSED = "rowsProcessed";
+  public static final String ROWS_PROCESSED_WITH_ERRORS = "rowsProcessedWithErrors";
+  public static final String ROWS_UNPARSEABLE = "rowsUnparseable";
+  public static final String ROWS_THROWN_AWAY = "rowsThrownAway";
+
+  public static Map<String, Object> makeIngestionRowMetrics(
+      long rowsProcessed,
+      long rowsProcessedWithErrors,
+      long rowsUnparseable,
+      long rowsThrownAway
+  )
+  {
+    Map<String, Object> metricsMap = Maps.newHashMap();
+    metricsMap.put(ROWS_PROCESSED, rowsProcessed);
+    metricsMap.put(ROWS_PROCESSED_WITH_ERRORS, rowsProcessedWithErrors);
+    metricsMap.put(ROWS_UNPARSEABLE, rowsUnparseable);
+    metricsMap.put(ROWS_THROWN_AWAY, rowsThrownAway);
+    return metricsMap;
+  }
+}

api/src/main/java/io/druid/indexer/TaskStatusPlus.java

@@ -38,6 +38,9 @@ public class TaskStatusPlus
   private final TaskLocation location;
   private final String dataSource;
 
+  @Nullable
+  private final String errorMsg;
+
   @JsonCreator
   public TaskStatusPlus(
       @JsonProperty("id") String id,
@@ -47,20 +50,22 @@ public TaskStatusPlus(
       @JsonProperty("statusCode") @Nullable TaskState state,
       @JsonProperty("duration") @Nullable Long duration,
       @JsonProperty("location") TaskLocation location,
-      @JsonProperty("dataSource") String dataSource
+      @JsonProperty("dataSource") @Nullable String dataSource, // nullable for backward compatibility
+      @JsonProperty("errorMsg") @Nullable String errorMsg
   )
   {
     if (state != null && state.isComplete()) {
       Preconditions.checkNotNull(duration, "duration");
     }
     this.id = Preconditions.checkNotNull(id, "id");
-    this.type = Preconditions.checkNotNull(type, "type");
+    this.type = type;
     this.createdTime = Preconditions.checkNotNull(createdTime, "createdTime");
     this.queueInsertionTime = Preconditions.checkNotNull(queueInsertionTime, "queueInsertionTime");
     this.state = state;
     this.duration = duration;
     this.location = Preconditions.checkNotNull(location, "location");
     this.dataSource = dataSource;
+    this.errorMsg = errorMsg;
   }
 
   @JsonProperty
@@ -108,49 +113,53 @@ public TaskLocation getLocation()
     return location;
   }
 
+  @JsonProperty
+  public String getDataSource()
+  {
+    return dataSource;
+  }
+
+  @Nullable
+  @JsonProperty("errorMsg")
+  public String getErrorMsg()
+  {
+    return errorMsg;
+  }
+
   @Override
   public boolean equals(Object o)
   {
     if (this == o) {
       return true;
     }
-
     if (o == null || getClass() != o.getClass()) {
       return false;
     }
-
-    final TaskStatusPlus that = (TaskStatusPlus) o;
-    if (!id.equals(that.id)) {
-      return false;
-    }
-    if (!type.equals(that.type)) {
-      return false;
-    }
-    if (!createdTime.equals(that.createdTime)) {
-      return false;
-    }
-    if (!queueInsertionTime.equals(that.queueInsertionTime)) {
-      return false;
-    }
-    if (!Objects.equals(state, that.state)) {
-      return false;
-    }
-    if (!Objects.equals(duration, that.duration)) {
-      return false;
-    }
-    return location.equals(that.location);
+    TaskStatusPlus that = (TaskStatusPlus) o;
+    return Objects.equals(getId(), that.getId()) &&
+           Objects.equals(getType(), that.getType()) &&
+           Objects.equals(getCreatedTime(), that.getCreatedTime()) &&
+           Objects.equals(getQueueInsertionTime(), that.getQueueInsertionTime()) &&
+           getState() == that.getState() &&
+           Objects.equals(getDuration(), that.getDuration()) &&
+           Objects.equals(getLocation(), that.getLocation()) &&
+           Objects.equals(getDataSource(), that.getDataSource()) &&
+           Objects.equals(getErrorMsg(), that.getErrorMsg());
   }
 
   @Override
   public int hashCode()
   {
-    return Objects.hash(id, type, createdTime, queueInsertionTime, state, duration, location);
-  }
-
-  @JsonProperty
-  public String getDataSource()
-  {
-    return dataSource;
+    return Objects.hash(
+        getId(),
+        getType(),
+        getCreatedTime(),
+        getQueueInsertionTime(),
+        getState(),
+        getDuration(),
+        getLocation(),
+        getDataSource(),
+        getErrorMsg()
+    );
   }
-
 }

api/src/main/java/io/druid/tasklogs/NoopTaskLogs.java

@@ -24,6 +24,7 @@
 import io.druid.java.util.common.logger.Logger;
 
 import java.io.File;
+import java.io.IOException;
 
 public class NoopTaskLogs implements TaskLogs
 {
@@ -41,6 +42,12 @@ public void pushTaskLog(String taskid, File logFile)
     log.info("Not pushing logs for task: %s", taskid);
   }
 
+  @Override
+  public void pushTaskReports(String taskid, File reportFile) throws IOException
+  {
+    log.info("Not pushing reports for task: %s", taskid);
+  }
+
   @Override
   public void killAll()
   {

api/src/main/java/io/druid/tasklogs/TaskLogPusher.java

@@ -31,4 +31,8 @@
 public interface TaskLogPusher
 {
   void pushTaskLog(String taskid, File logFile) throws IOException;
+
+  default void pushTaskReports(String taskid, File reportFile) throws IOException
+  {
+  }
 }

api/src/main/java/io/druid/tasklogs/TaskLogStreamer.java

@@ -40,4 +40,9 @@ public interface TaskLogStreamer
    * @return input supplier for this log, if available from this provider
    */
   Optional<ByteSource> streamTaskLog(String taskid, long offset) throws IOException;
+
+  default Optional<ByteSource> streamTaskReports(final String taskid) throws IOException
+  {
+    return Optional.absent();
+  }
 }

api/src/main/java/io/druid/timeline/partition/NoneShardSpec.java

@@ -22,7 +22,7 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Range;
+import com.google.common.collect.RangeSet;
 import io.druid.data.input.InputRow;
 
 import java.util.List;
@@ -71,19 +71,11 @@ public int getPartitionNum()
   @Override
   public ShardSpecLookup getLookup(final List<ShardSpec> shardSpecs)
   {
-
-    return new ShardSpecLookup()
-    {
-      @Override
-      public ShardSpec getShardSpec(long timestamp, InputRow row)
-      {
-        return shardSpecs.get(0);
-      }
-    };
+    return (long timestamp, InputRow row) -> shardSpecs.get(0);
   }
 
   @Override
-  public Map<String, Range<String>> getDomain()
+  public Map<String, RangeSet<String>> getDomain()
   {
     return ImmutableMap.of();
   }

api/src/main/java/io/druid/timeline/partition/ShardSpec.java

@@ -21,7 +21,7 @@
 
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import com.google.common.collect.Range;
+import com.google.common.collect.RangeSet;
 import io.druid.data.input.InputRow;
 
 import java.util.List;
@@ -50,5 +50,5 @@ public interface ShardSpec
    *
    * @return map of dimensions to its possible range. Dimensions with unknown possible range are not mapped
    */
-  Map<String, Range<String>> getDomain();
+  Map<String, RangeSet<String>> getDomain();
 }