cd /usr/local/src/collectd-fast-jmx-20150603

vi enhance-lookupAttribute.patch  # Lots of tabs in the patch -- omit the ------ lines
------
--- ../collectd-fast-jmx-master/src/main/java/org/collectd/AttributePermutation.java    2015-06-03 09:48:21.000000000 -0400
+++ src/main/java/org/collectd/AttributePermutation.java    2015-07-12 23:39:58.000000000 -0400
@@ -10,11 +10,13 @@
 import javax.management.ObjectName;
 import javax.management.openmbean.CompositeData;
 import javax.management.openmbean.OpenType;
+import javax.management.openmbean.TabularData;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Collection;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Callable;
@@ -223,6 +225,56 @@
                            value = compositeValue.get(node);
                        } else if (value instanceof OpenType) {
                            throw new UnsupportedOperationException("Handling of OpenType " + ((OpenType) value).getTypeName() + " is not yet implemented.");
+                       } else if (value instanceof TabularData) {
+                           // A java.lang:type=GarbageCollector mbean
+                           // has an LastGcInfo attribute that is a
+                           // CompositeData value, which interesting
+                           // sub-value called memoryUsageAfterGc and
+                           // memoryUsageBeforeGc, which expose the
+                           // javax.management.openmbean.TabularData
+                           // interface. Each table exposes each heap
+                           // and nonheap memory pool using the name
+                           // of its java.lang:type=MemoryPool as the
+                           // TabularData key. The memory pool is a
+                           // CompositeData value that can be further
+                           // examined with Attribute dot notation.
+                           //   #mbean = java.lang:type=GarbageCollector,name=ParNew
+                           //   LastGcInfo = { 
+                           //     memoryUsageAfterGc = {
+                           //       ( Par Survivor Space ) = {
+                           //         key = Par Survivor Space;
+                           //         value = {
+                           //           committed = 8716288;
+                           //           init = 8716288;
+                           //           max = 8716288;
+                           //           used = 79040;
+                           //          };
+                           //        };
+                           //       ...
+                           //      };
+                           //    };
+                           //
+                           // This java.conf entry would capture the state
+                           // of the "Par Eden Space" memory pool after the
+                           // most recent garbage collection invation:
+                           //    <MBean "java/garbage_collector">
+                           //      ObjectName "java.lang:type=GarbageCollector,*"
+                           //      <Value>
+                           //        Attribute "LastGcInfo.memoryUsageAfterGc.Par Eden Space"
+                           //        Table true
+                           //        Type "memory" # value:GAUGE:0:281474976710656
+                           //        InstancePrefix "pool-eden-after"
+                           //      </Value>
+                           //    </MBean>
+
+                           TabularData tabularData = (TabularData) value;
+                           Collection<CompositeData> table =
+                               (Collection<CompositeData>)tabularData.values();
+                           for (CompositeData compositeData : table) {
+                               if (node.equals(compositeData.get("key"))) {
+                                   value = compositeData.get("value");
+                               }
+                           }
                        } else if (value != null) {
                            // Try to traverse via Reflection.
                            value = value.getClass().getDeclaredField(node).get(value);
@@ -294,6 +346,7 @@
            //
            consecutiveNotFounds++;
        } catch (Exception ex) {
+                   logger.warning("catch: " + ex);
            throw ex;
        } finally {
            lastRunDuration = System.nanoTime() - start;
------

patch -p0 <enhance-lookupAttribute.patch

import java.io.BufferedReader;
import java.io.InputStreamReader;

        private void reportNetstat(String label)
        {
          final String cmd = "netstat -an";
          try {
              // Run netstat
              Process process = Runtime.getRuntime().exec(cmd);
              process.waitFor();
              BufferedReader reader =
                   new BufferedReader(new InputStreamReader(process.getInputStream()));
              String line = "";
              while ((line = reader.readLine())!= null) {
                if (line.contains(":10001      ESTABLISHED")
                 || line.contains(":10002      ESTABLISHED")
                 || line.contains(":10001      TIME_WAIT")
                 || line.contains(":10002      TIME_WAIT")
                ) {
                  logger.info("NETSTAT " + label + ": " + line);
                }
              }
          } catch (Exception e) {
              e.printStackTrace(System.err);
          }
        }

cd /tmp/mina

cat >>TCPServer.java <<EOF
import java.io.*;
import java.net.*;
class TCPServer {
    public static void main(String argv[]) throws Exception       {
         String clientSentence;
         String capitalizedSentence;
         int portNumber = Integer.valueOf(argv[0]);
         ServerSocket welcomeSocket = new ServerSocket(portNumber);
         while(true)          {
             Socket connectionSocket = welcomeSocket.accept();
            BufferedReader inFromClient =                new BufferedReader(new InputStreamReader(connectionSocket.getInputStream()));
            DataOutputStream outToClient = new DataOutputStream(connectionSocket.getOutputStream());
            clientSentence = inFromClient.readLine();
            System.out.println("Received: " + clientSentence);
            capitalizedSentence = clientSentence.toUpperCase() + '\n';
            outToClient.writeBytes(capitalizedSentence);
         }
       }
 }
EOF

javac *java

java -Dcom.sun.management.jmxremote.port=10001 \
     -Dcom.sun.management.jmxremote.local.only=false \
     -Dcom.sun.management.jmxremote.authenticate=false \
     -Dcom.sun.management.jmxremote.ssl=false \
     -classpath . TCPServer 20001

[2015-07-09 18:49:36] FastJMX Plugin: Failed to collect: java.lang:type=MemoryPool,name=PS Survivor  Space@service:jmx:rmi:///jndi/rmi://127.0.0.1:8855/jmxrmi InstanceNotFound consecutive count=5

 LoadPlugin java
 <Plugin "java">
 JVMARG "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/collectd-fast-jmx-1.0.0.jar:/usr/share/collectd/java/jconsole.jar:/usr/share/collectd/java/jmxremote_optional.jar"
 LoadPlugin "com.e_gineering.collectd.FastJMX"

 <Plugin "FastJMX">
 MaxThreads 256
 CollectInternal true
LogLevel "FINE"
ForceLoggingTo "WARNING"
# Memory usage by memory pool.
<MBean "memory_pool">
  ObjectName "java.lang:type=MemoryPool,*"
  InstancePrefix "memory_pool-"
  InstanceFrom "name"

  <Value "Usage">
    Type "java_memory"
    Composite true
    PluginName "JVM"
  </Value>
</MBean>

<Connection>
  ServiceURL "service:jmx:jmxmp://localhost:31301"
  IncludePortInHostname true
  Username "foo"
  Password "bar"
  Collect "memory_pool"
</Connection>

SEVERE: Cannot collect internal metrics. Please ensure types.db contains: 'fastjmx_latency  value:GAUGE:0:U'.

LoadPlugin java

<Plugin "java">

 JVMARG "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/collectd-fast-jmx.jar"

  LoadPlugin "com.e_gineering.collectd.FastJMX"

  <Plugin "FastJMX">

    MaxThreads 256
    CollectInternal true

    <MBean "classes">
      ObjectName "java.lang:type=ClassLoading"

      <Value "LoadedClassCount">
        Type "gauge"
        InstancePrefix "loaded_classes"
        PluginName "JVM"
      </Value>
    </MBean>
...................

 collectd[19249]: java plugin: cjni_config_load_plugin: FindClass (com/e_gineering/collectd/FastJMX) failed.
Jan 26 20:14:19  collectd: Exception in thread "main" java.lang.NoClassDefFoundError: com/e_gineering/collectd/FastJMX
Jan 26 20:14:19  collectd: Caused by: java.lang.ClassNotFoundException: com.e_gineering.collectd.FastJMX
Jan 26 20:14:19  collectd: at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
Jan 26 20:14:19  collectd: at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
Jan 26 20:14:19  collectd: at java.security.AccessController.doPrivileged(Native Method)
Jan 26 20:14:19  collectd[19249]: java plugin: Configuration block for `FastJMX' found, but no such configuration callback has been registered. Please make sure, the `LoadPlugin' lines precede the `Plugin' blocks.
Jan 26 20:14:19  collectd: at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
Jan 26 20:14:19  collectd: at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
Jan 26 20:14:19  collectd: at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
Jan 26 20:14:19  collectd: at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
Jan 26 20:14:19  collectd[19249]: Initialization complete, entering read-loop.

Host "hostname[hostHasService=activemq]"

LoadPlugin java

<Plugin java>
  JVMARG "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/collectd-fast-jmx-1.0.0.jar"

  LoadPlugin "com.e_gineering.collectd.FastJMX"

  <Plugin FastJMX>
    LogLevel "FINE"
    MaxThreads 256

    <MBean "memory">
      ObjectName "java.lang:type=Memory"

      <Value>
        Attribute "HeapMemoryUsage"
        Type "memory"
        Composite true
        InstancePrefix "heap"
        PluginName "jvm"
      </Value>
      <Value>
        Attribute "NonHeapMemoryUsage"
        Type "memory"
        Composite true
        InstancePrefx "non-heap"
        PluginName "jvm"
      </Value>
    </MBean>

    <Connection "cass-test-1">
      ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:7199/jmxrmi"
      Collect "memory"
    </Connection>

  </Plugin>
</Plugin>

LoadPlugin java
<Plugin "java">
  JVMARG "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/fast-jmx.jar:/usr/share/collectd/java/jboss-client.jar"
  LoadPlugin "org.collectd.FastJMX"
  <Plugin "FastJMX">
    LogLevel "FINEST"
    ForceLoggingTo "INFO"
    CollectInternal true

    <MBean "Service1_ClassLoading">
      ObjectName "java.lang:type=ClassLoading"
      InstancePrefix "ClassLoading"
      <Value>
        Type "gauge"
        Table false
        InstancePrefix "ClassesTotalLoadedClassCount"
        Attribute "TotalLoadedClassCount"
        PluginName 'Service1'
      </Value>
      <Value>
        Type "gauge"
        Table false
        InstancePrefix "ClassesUnloadedClassCount"
        Attribute "UnloadedClassCount"
        PluginName 'Service1'
      </Value>
    </MBean>

    <Connection>
      ServiceURL "service:jmx:rmi:///jndi/rmi://{{CONTAINER_HOST_ADDRESS}}:9000/jmxrmi"
      Synchronous true
      Host "{{HOST_NAME}}"
      InstancePrefix 'JVM'
      Collect "Service1_ClassLoading"
    </Connection>
    {% endfor %}
    {% endif %}

    <MBean "Service2_ClassLoading">
      ObjectName "java.lang:type=ClassLoading"
      InstancePrefix "ClassLoading"
      <Value>
        Type "gauge"
        Table false
        InstancePrefix "ClassesTotalLoadedClassCount"
        Attribute "TotalLoadedClassCount"
        PluginName 'Service2'
      </Value>
      <Value>
        Type "gauge"
        Table false
        InstancePrefix "ClassesUnloadedClassCount"
        Attribute "UnloadedClassCount"
        PluginName 'Service2'
      </Value>
    </MBean>

    <Connection>
      ServiceURL  "service:jmx:http-remoting-jmx://{{CONTAINER_HOST_ADDRESS}}:{{JMX_PORT}}"
      User "admin"
      Password "admin"
      Host "{{HOST_NAME}}"
      InstancePrefix 'JVM'
      Collect "Service2_ClassLoading"
    </Connection>
  </Plugin>
</Plugin>

Jul 10, 2015 9:57:45 AM org.collectd.Connection close
WARNING: Exception closing JMXConnection: error during JRMP connection establishment; nested exception is:
        java.net.SocketTimeoutException: Read timed out
Jul 10, 2015 9:57:50 AM org.collectd.SelfTuningCollectionExecutor push
WARNING: Failed to collect 13 of 13 samples within read interval with 1 threads.
Jul 10, 2015 9:57:50 AM org.collectd.Connection$ReconnectTask run
SEVERE: Failure to close for TTL reconnect to: service:jmx:rmi:///jndi/rmi://HOSTNAME:PORT/jmxrmi
Jul 10, 2015 9:57:50 AM org.collectd.Connection$ReconnectTask run
INFO: Error or TTL Expiration for service:jmx:rmi:///jndi/rmi://HOSTNAME:PORT/jmxrmi forcing reconnect..
Exception in thread "Connect-service:jmx:rmi:///jndi/rmi://HOSTNAME:PORT/jmxrmi" java.lang.NullPointerException
        at org.collectd.Connection$ReconnectTask.run(Connection.java:202)
        at java.util.TimerThread.mainLoop(Timer.java:555)
        at java.util.TimerThread.run(Timer.java:505)