Add metrics for SCHEDULED_TO_STAR latency (#380)

Author: vancexu

src/main/java/com/uber/cadence/internal/metrics/MetricsType.java

@@ -53,6 +53,8 @@ public class MetricsType {
       CADENCE_METRICS_PREFIX + "decision-poll-succeed";
   public static final String DECISION_POLL_LATENCY =
       CADENCE_METRICS_PREFIX + "decision-poll-latency"; // measure succeed poll request latency
+  public static final String DECISION_SCHEDULED_TO_START_LATENCY =
+      CADENCE_METRICS_PREFIX + "decision-scheduled-to-start-latency";
   public static final String DECISION_EXECUTION_FAILED_COUNTER =
       CADENCE_METRICS_PREFIX + "decision-execution-failed";
   public static final String DECISION_EXECUTION_LATENCY =
@@ -77,6 +79,8 @@ public class MetricsType {
       CADENCE_METRICS_PREFIX + "activity-poll-succeed";
   public static final String ACTIVITY_POLL_LATENCY =
       CADENCE_METRICS_PREFIX + "activity-poll-latency";
+  public static final String ACTIVITY_SCHEDULED_TO_START_LATENCY =
+      CADENCE_METRICS_PREFIX + "activity-scheduled-to-start-latency";
   public static final String ACTIVITY_EXEC_FAILED_COUNTER =
       CADENCE_METRICS_PREFIX + "activity-execution-failed";
   public static final String ACTIVITY_EXEC_LATENCY =
@@ -120,7 +124,6 @@ public class MetricsType {
 
   public static final String UNHANDLED_SIGNALS_COUNTER =
       CADENCE_METRICS_PREFIX + "unhandled-signals";
-
   public static final String CORRUPTED_SIGNALS_COUNTER =
       CADENCE_METRICS_PREFIX + "corrupted-signals";
 

src/main/java/com/uber/cadence/internal/worker/ActivityPollTask.java

@@ -26,6 +26,7 @@
 import com.uber.cadence.internal.metrics.MetricsType;
 import com.uber.cadence.serviceclient.IWorkflowService;
 import com.uber.m3.tally.Stopwatch;
+import com.uber.m3.util.Duration;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -91,6 +92,10 @@ public ActivityWorker.MeasurableActivityTask poll() throws TException {
     }
 
     options.getMetricsScope().counter(MetricsType.ACTIVITY_POLL_SUCCEED_COUNTER).inc(1);
+    options
+        .getMetricsScope()
+        .timer(MetricsType.ACTIVITY_SCHEDULED_TO_START_LATENCY)
+        .record(Duration.ofNanos(result.getStartedTimestamp() - result.getScheduledTimestampOfThisAttempt()));
     sw.stop();
     return new ActivityWorker.MeasurableActivityTask(result, e2eSW);
   }

src/main/java/com/uber/cadence/internal/worker/WorkflowPollTask.java

@@ -26,6 +26,7 @@
 import com.uber.cadence.serviceclient.IWorkflowService;
 import com.uber.m3.tally.Scope;
 import com.uber.m3.tally.Stopwatch;
+import com.uber.m3.util.Duration;
 import java.util.Objects;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
@@ -101,6 +102,9 @@ public PollForDecisionTaskResponse poll() throws TException {
     }
 
     metricScope.counter(MetricsType.DECISION_POLL_SUCCEED_COUNTER).inc(1);
+    metricScope
+        .timer(MetricsType.DECISION_SCHEDULED_TO_START_LATENCY)
+        .record(Duration.ofNanos(result.getStartedTimestamp() - result.getScheduledTimestamp()));
     sw.stop();
     return result;
   }