Log all available jmx metrics at debug level (#1826)

* Add option to log available jmx metrics

* Spotless

* Change from special config to normal debug logging

* Reduce log level for expected errors

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/TelemetryClientInitializer.java

@@ -65,6 +65,10 @@ public static void initialize(TelemetryClient telemetryClient, Configuration con
     PerformanceCounterContainer.INSTANCE.setCollectionFrequencyInSec(
         configuration.preview.metricIntervalSeconds);
 
+    if (logger.isDebugEnabled()) {
+      PerformanceCounterContainer.INSTANCE.setLogAvailableJmxMetrics();
+    }
+
     loadCustomJmxPerfCounters(configuration.jmxMetrics);
 
     PerformanceCounterContainer.INSTANCE.register(new ProcessCpuPerformanceCounter());

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/perfcounter/AvailableJmxMetricLogger.java

@@ -0,0 +1,248 @@
+/*
+ * ApplicationInsights-Java
+ * Copyright (c) Microsoft Corporation
+ * All rights reserved.
+ *
+ * MIT License
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the ""Software""), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in all copies or
+ * substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+package com.microsoft.applicationinsights.agent.internal.perfcounter;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanInfo;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.CompositeDataSupport;
+import javax.management.openmbean.CompositeType;
+import javax.management.openmbean.OpenType;
+import org.checkerframework.checker.lock.qual.GuardedBy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class AvailableJmxMetricLogger {
+
+  private static final Logger logger = LoggerFactory.getLogger(AvailableJmxMetricLogger.class);
+
+  private static final Set<String> NUMERIC_ATTRIBUTE_TYPES =
+      new HashSet<>(
+          asList(
+              "long",
+              "int",
+              "double",
+              "float",
+              "java.lang.Long",
+              "java.lang.Integer",
+              "java.lang.Double",
+              "java.lang.Float"));
+
+  private static final Set<String> BOOLEAN_ATTRIBUTE_TYPES =
+      new HashSet<>(asList("boolean", "java.lang.Boolean"));
+
+  @GuardedBy("lock")
+  private Map<String, Set<String>> priorAvailableJmxAttributes = new HashMap<>();
+
+  private final Object lock = new Object();
+
+  void logAvailableJmxMetrics() {
+    synchronized (lock) {
+      Map<String, Set<String>> availableJmxAttributes = getAvailableJmxAttributes();
+      logDifference(priorAvailableJmxAttributes, availableJmxAttributes);
+      priorAvailableJmxAttributes = availableJmxAttributes;
+    }
+  }
+
+  private static void logDifference(
+      Map<String, Set<String>> priorAvailableJmxAttributes,
+      Map<String, Set<String>> currentAvailableJmxAttributes) {
+    if (priorAvailableJmxAttributes.isEmpty()) {
+      // first time
+      logger.info("available jmx metrics:\n{}", toString(currentAvailableJmxAttributes));
+      return;
+    }
+    Map<String, Set<String>> newlyAvailable =
+        difference(currentAvailableJmxAttributes, priorAvailableJmxAttributes);
+    if (!newlyAvailable.isEmpty()) {
+      logger.info("newly available jmx metrics since last output:\n{}", toString(newlyAvailable));
+    }
+    Map<String, Set<String>> noLongerAvailable =
+        difference(priorAvailableJmxAttributes, currentAvailableJmxAttributes);
+    if (!noLongerAvailable.isEmpty()) {
+      logger.info(
+          "no longer available jmx metrics since last output:\n{}", toString(noLongerAvailable));
+    }
+  }
+
+  private static String toString(Map<String, Set<String>> jmxAttributes) {
+    StringBuilder sb = new StringBuilder();
+    for (Map.Entry<String, Set<String>> entry : jmxAttributes.entrySet()) {
+      sb.append("  - object name:        ")
+          .append(entry.getKey())
+          .append("\n")
+          .append("    numeric attributes: ")
+          .append(entry.getValue().stream().sorted().collect(Collectors.joining(", ")))
+          .append("\n");
+    }
+    return sb.toString();
+  }
+
+  private static Map<String, Set<String>> getAvailableJmxAttributes() {
+    MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+    Set<ObjectName> objectNames = server.queryNames(null, null);
+    Map<String, Set<String>> availableJmxMetrics = new HashMap<>();
+    for (ObjectName objectName : objectNames) {
+      String name = objectName.toString();
+      try {
+        Set<String> attrs = getJmxAttributes(server, objectName);
+        if (!attrs.isEmpty()) {
+          availableJmxMetrics.put(name, attrs);
+        }
+      } catch (Exception e) {
+        // log exception at trace level since this is expected in several cases, e.g.
+        // "java.lang.UnsupportedOperationException: CollectionUsage threshold is not supported"
+        // and available jmx metrics are already only logged at debug
+        logger.trace(e.getMessage(), e);
+        availableJmxMetrics.put(name, Collections.singleton("<error getting attributes: " + e));
+      }
+    }
+    return availableJmxMetrics;
+  }
+
+  private static Set<String> getJmxAttributes(MBeanServer server, ObjectName objectName)
+      throws Exception {
+    MBeanInfo mbeanInfo = server.getMBeanInfo(objectName);
+    Set<String> attributeNames = new HashSet<>();
+    for (MBeanAttributeInfo attribute : mbeanInfo.getAttributes()) {
+      if (attribute.isReadable()) {
+        try {
+          Object value = server.getAttribute(objectName, attribute.getName());
+          attributeNames.addAll(getNumericAttributes(attribute, value));
+        } catch (Exception e) {
+          // log exception at trace level since this is expected in several cases, e.g.
+          // "java.lang.UnsupportedOperationException: CollectionUsage threshold is not supported"
+          // and available jmx metrics are already only logged at debug
+          logger.trace(e.getMessage(), e);
+        }
+      }
+    }
+    return attributeNames;
+  }
+
+  private static List<String> getNumericAttributes(MBeanAttributeInfo attribute, Object value) {
+    String attributeType = attribute.getType();
+    if (NUMERIC_ATTRIBUTE_TYPES.contains(attributeType) && value instanceof Number) {
+      return singletonList(attribute.getName());
+    }
+    if (BOOLEAN_ATTRIBUTE_TYPES.contains(attributeType) && value instanceof Boolean) {
+      return singletonList(attribute.getName());
+    }
+    if (attributeType.equals("java.lang.Object") && value instanceof Number) {
+      return singletonList(attribute.getName());
+    }
+    if (attributeType.equals("java.lang.String") && value instanceof String) {
+      try {
+        Double.parseDouble((String) value);
+        return singletonList(attribute.getName());
+      } catch (NumberFormatException e) {
+        // this is expected for non-numeric attributes
+        return emptyList();
+      }
+    }
+    if (attributeType.equals(CompositeData.class.getName())) {
+      Object openType = attribute.getDescriptor().getFieldValue("openType");
+      CompositeType compositeType = null;
+      if (openType instanceof CompositeType) {
+        compositeType = (CompositeType) openType;
+      } else if (openType == null && value instanceof CompositeDataSupport) {
+        compositeType = ((CompositeDataSupport) value).getCompositeType();
+      }
+      if (compositeType != null) {
+        return getCompositeTypeAttributeNames(attribute, value, compositeType);
+      }
+    }
+    return emptyList();
+  }
+
+  private static List<String> getCompositeTypeAttributeNames(
+      MBeanAttributeInfo attribute, Object compositeData, CompositeType compositeType) {
+    List<String> attributeNames = new ArrayList<>();
+    for (String itemName : compositeType.keySet()) {
+      OpenType<?> itemType = compositeType.getType(itemName);
+      if (itemType == null) {
+        continue;
+      }
+      String className = itemType.getClassName();
+      Class<?> clazz;
+      try {
+        clazz = Class.forName(className);
+      } catch (ClassNotFoundException e) {
+        logger.warn(e.getMessage(), e);
+        continue;
+      }
+      if (Number.class.isAssignableFrom(clazz)) {
+        attributeNames.add(attribute.getName() + '.' + itemName);
+      } else if (clazz == String.class && compositeData instanceof CompositeData) {
+        Object val = ((CompositeData) compositeData).get(itemName);
+        if (val instanceof String) {
+          try {
+            Double.parseDouble((String) val);
+            attributeNames.add(attribute.getName() + '.' + itemName);
+          } catch (NumberFormatException e) {
+            // this is expected for non-numeric attributes
+          }
+        }
+      }
+    }
+    return attributeNames;
+  }
+
+  // visible for testing
+  static Map<String, Set<String>> difference(
+      Map<String, Set<String>> map1, Map<String, Set<String>> map2) {
+    Map<String, Set<String>> difference = new HashMap<>();
+    for (Map.Entry<String, Set<String>> entry : map1.entrySet()) {
+      String key = entry.getKey();
+      Set<String> diff = difference(entry.getValue(), map2.get(key));
+      if (!diff.isEmpty()) {
+        difference.put(entry.getKey(), diff);
+      }
+    }
+    return difference;
+  }
+
+  private static Set<String> difference(Set<String> set1, @Nullable Set<String> set2) {
+    if (set2 == null) {
+      return set1;
+    }
+    HashSet<String> difference = new HashSet<>(set1);
+    difference.removeAll(set2);
+    return difference;
+  }
+}

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/perfcounter/PerformanceCounterContainer.java

@@ -27,6 +27,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import org.checkerframework.checker.nullness.qual.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -64,6 +65,8 @@ public enum PerformanceCounterContainer {
   private final ConcurrentMap<String, PerformanceCounter> performanceCounters =
       new ConcurrentHashMap<>();
 
+  private volatile @Nullable AvailableJmxMetricLogger availableJmxMetricLogger;
+
   private volatile boolean initialized = false;
 
   private long collectionFrequencyInMillis = DEFAULT_COLLECTION_FREQUENCY_IN_SEC * 1000;
@@ -117,6 +120,10 @@ public void setCollectionFrequencyInSec(long collectionFrequencyInSec) {
     this.collectionFrequencyInMillis = collectionFrequencyInSec * 1000;
   }
 
+  public void setLogAvailableJmxMetrics() {
+    availableJmxMetricLogger = new AvailableJmxMetricLogger();
+  }
+
   /**
    * A private method that is called only when the container needs to start collecting performance
    * counters data. The method will schedule a callback to be called, it will initialize a {@link
@@ -141,6 +148,10 @@ private void scheduleWork() {
         new Runnable() {
           @Override
           public void run() {
+            if (availableJmxMetricLogger != null) {
+              availableJmxMetricLogger.logAvailableJmxMetrics();
+            }
+
             TelemetryClient telemetryClient = TelemetryClient.getActive();
 
             for (PerformanceCounter performanceCounter : performanceCounters.values()) {

agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/perfcounter/AvailableJmxMetricLoggerTest.java

@@ -0,0 +1,64 @@
+/*
+ * ApplicationInsights-Java
+ * Copyright (c) Microsoft Corporation
+ * All rights reserved.
+ *
+ * MIT License
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the ""Software""), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in all copies or
+ * substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+package com.microsoft.applicationinsights.agent.internal.perfcounter;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.singleton;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.junit.jupiter.api.Test;
+
+public class AvailableJmxMetricLoggerTest {
+
+  @Test
+  public void testDifference() {
+    // given
+    Map<String, Set<String>> map1 = new HashMap<>();
+    map1.put("one", singleton("1"));
+    map1.put("two", new HashSet<>(asList("2", "22")));
+    map1.put("three", new HashSet<>(asList("3", "33", "333")));
+
+    Map<String, Set<String>> map2 = new HashMap<>();
+    map2.put("one", singleton("1"));
+    map2.put("two", singleton("22"));
+
+    // when
+    Map<String, Set<String>> difference = AvailableJmxMetricLogger.difference(map1, map2);
+
+    // then
+    assertThat(difference).containsOnlyKeys("two", "three");
+    assertThat(difference.get("two")).containsExactly("2");
+    assertThat(difference.get("three")).containsExactlyInAnyOrder("3", "33", "333");
+  }
+
+  @Test
+  public void test() {
+    AvailableJmxMetricLogger availableJmxMetricLogger = new AvailableJmxMetricLogger();
+
+    availableJmxMetricLogger.logAvailableJmxMetrics();
+    availableJmxMetricLogger.logAvailableJmxMetrics();
+  }
+}