This post is quite succinct as I consider you already know Ganglia, Python and Java.

To monitor Java JMX values with Ganglia, I used JPype. JPype is an effort to allow python programs full access to java class libraries. This allowed me to port easily the JMX Munin plugin, you can get this plugin on Munin Exchange. The port, is really basic as I just converted few method to make sure I can publicly access them from the Python module. You can take a look at the jmxquery.patch or download jmxquery.jar.

diff -urBN jmxquery.a/src/jmxquery/src/org/munin/JMXQuery.java jmxquery.b/src/jmxquery/src/org/munin/JMXQuery.java
--- jmxquery.a/src/jmxquery/src/org/munin/JMXQuery.java	2007-05-14 01:07:44.000000000 -0700
+++ jmxquery.b/src/jmxquery/src/org/munin/JMXQuery.java	2010-01-15 15:40:00.000000000 -0800
@@ -61,7 +61,7 @@
 		this.password = password;
 	}

-	private void connect() throws IOException
+	public void connect() throws IOException
 	{
 		Map environment = null;
 		if (username != null && password != null)
@@ -78,7 +78,18 @@
 		connection = connector.getMBeanServerConnection();
 	}

-	private void list() throws IOException, InstanceNotFoundException, IntrospectionException, ReflectionException
+	public void ping() throws IOException
+	{
+	  try {
+	    MBeanServerConnection testConnection = connector.getMBeanServerConnection();
+	    if (testConnection == null)
+	       this.connect();
+	  } catch (IOException e) {
+	    this.connect();
+	  }
+	}
+
+	public void list() throws IOException, InstanceNotFoundException, IntrospectionException, ReflectionException
 	{
 		if (config == null)
 		{
@@ -90,7 +101,7 @@
 		}
 	}

-	private void listConfig()
+	public void listConfig()
 	{
 		for (FieldProperties field : config.getFields())
 		{
@@ -107,7 +118,22 @@
 		}
 	}

-	private void output(String name, Object attr, String key)
+	public String output(String name, String JmxObjectName, String JmxAttributeName, String Key)
+	{
+    try
+    {
+      Object value = connection.getAttribute(new ObjectName(JmxObjectName), JmxAttributeName);
+      return output(name, value, Key);
+    }
+    catch (Exception e)
+    {
+      System.err.println("Fail to output " + name);
+      e.printStackTrace();
+    }
+    return null;
+	}
+
+	public String output(String name, Object attr, String key)
 	{
 		if (attr instanceof CompositeDataSupport)
 		{
@@ -116,15 +142,17 @@
 			{
 				throw new IllegalArgumentException("Key is null for composed data " + name);
 			}
-			System.out.println(name + ".value " + format(cds.get(key)));
+			//System.out.println(name + ".value " + format(cds.get(key)));
+			return format(cds.get(key));
 		}
 		else
 		{
-			System.out.println(name + ".value " + format(attr));
+			//System.out.println(name + ".value " + format(attr));
+		  return format(attr);
 		}
 	}

-	private void output(String name, Object attr)
+	public void output(String name, Object attr)
 	{
 		if (attr instanceof CompositeDataSupport)
 		{
@@ -142,7 +170,7 @@
 	}

 	@SuppressWarnings("unchecked")
-	private void listAll() throws IOException, InstanceNotFoundException, IntrospectionException, ReflectionException
+	public void listAll() throws IOException, InstanceNotFoundException, IntrospectionException, ReflectionException
 	{
 		Set

Once JPype is installed, make sure your Ganglia is configured with the python module.

# The modules section describes the module
#  that should be loaded.
#   name - module name
#   path - load path of the .so
#   params - path to the directory where mod_python
#             should look for python metric modules
modules {
  module {
    name = "python_module"
    path = "modpython.so"
    params = "/opt/ganglia/python_modules"
  }
}

include ('/opt/ganglia/etc/conf.d/*.pyconf')

Then you can use this basic jmx python module:

"""
Ganglia Module to graph JMX values using JPype
Author: Nicolas Brousse
"""
from jpype import *

import os, platform, sys, time

debug = 0

_jmx = None
_jmx_cur_status = {}
_jmx_pre_status = {}
_jmx_pre_time = {}
_eta = 0
_jmx_params = {'User': '',
              'Password': '',
              'Host': '127.0.0.1',
              'Port': '8989',
              'JMXWrapper': '/opt/ganglia/jmxquery.jar'}

descriptors = []

gauge_metrics = {}

def Metric_Handler(name):
    global _jmx

    if _jmx is None:
        _jmx = initJVM()

    pre = None
    cur = 0
    eta = 0

    if debug:
        print "[ DEBUG ] _jmx_cur_status length: %d" % len(_jmx_cur_status)
        print "[ DEBUG ] _jmx_pre_status length: %d" % len(_jmx_pre_status)

    if _jmx is None:
        raise TypeError("No valid JMX Object !")

    if name in _jmx_cur_status:
        _jmx_pre_status[name] = _jmx_cur_status[name]

    for d in descriptors:
        if d['name'] == name:
            break

    try:
        if "jmxKeyName" in d:
            key = d['jmxKeyName']
        else:
            key = None

        if debug:
            print "[ DEBUG ] _jmx.output(%s, %s, %s, %s)" % \
                (d['name'], d['jmxObjectName'], d['jmxAttributeName'], key)

        _jmx.ping()
        result = _jmx.output(d['name'], d['jmxObjectName'], d['jmxAttributeName'], key)
    except:
        print "Error: ", sys.exc_info()[1]
        raise

    if result.isdigit():
        if debug: print "[ DEBUG ] %s: %s" % (name, result)
        _jmx_cur_status[name] = result

        cur_time = time.time()
        if name in _jmx_pre_time:
            eta = cur_time - _jmx_pre_time[name]
            if debug:
                print '[ DEBUG ] ETA: %d, Last time: %d, Cur time: %d' % (eta, _jmx_pre_time[name], cur_time)

        _jmx_pre_time[name] = cur_time

    if name in _jmx_cur_status:
        cur = _jmx_cur_status[name]
    else:
        cur = 0
    if len(_jmx_pre_status) > 0 and name in _jmx_pre_status:
        pre = _jmx_pre_status[name]
    else:
        pre = 0
    eta = int(eta)

    if name in gauge_metrics:
        if pre is not None and eta > 0:
            ret = (long(cur) - long(pre)) / eta
        else:
            ret = 0
    else:
        ret = cur

    if debug:
        print "[ DEBUG ] Metric_Handler(%s): %s (eta: %s, cur: %s, pre: %s)" % (name, ret, eta, cur, pre)

    return long(ret)

def Init_Metric (name, jmxObjectName, jmxAttributeName, jmxKeyName, tmax, type, slope, units, fmt, handler):
    '''Create a metric definition dictionary object.'''
    d = {'name': name.lower(),
        'jmxObjectName': jmxObjectName,
        'jmxAttributeName': jmxAttributeName,
        'jmxKeyName': jmxKeyName,
        'call_back': handler,
        'time_max': tmax,
        'value_type': type,
        'units': units,
        'slope': slope,
        'format': fmt,
        'description': 'JMX '+name.replace("_"," "),
	    'groups': 'jmx'}
    return d

def metric_init(params):
    global _jmx, _jmx_params

    if debug: print "[ DEBUG ] metric_init()"

    if 'User' in params:
        _jmx_params['User'] = params['User']

    if 'Password' in params:
        _jmx_params['Password'] = params['Password']

    if 'Host' in params:
        _jmx_params['Host'] = params['Host']

    if 'Port' in params:
        _jmx_params['Port'] = int(params['Port'])

    if 'JMXWrapper' in params:
        _jmx_params['JMXWrapper'] = params['JMXWrapper']

    return descriptors

def initJVM():
    if debug: print "[ DEBUG ] initJVM()"

    if platform.architecture()[0] == "32bit":
        arch="i386"
    else:
        arch="amd64"

    jvm="/usr/java/jdk/jre/lib/%s/client/libjvm.so" % arch
    if not os.path.isfile(jvm):
        jvm="/usr/java/jdk/jre/lib/%s/server/libjvm.so" % arch
        if not os.path.isfile(jvm):
            raise IOError("Can't find the libjvm.so (%s)" % jvm)

    if not os.path.isfile(_jmx_params['JMXWrapper']):
        raise IOError("Can't find the JMX Wrapper at'%s'" % \
            _jmx_params['JMXWrapper'])

    try:
        startJVM(jvm, "-Djava.class.path=%s" % _jmx_params['JMXWrapper'])

        url = "service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi" % \
               (_jmx_params['Host'], int(_jmx_params['Port']))

        JMXQuery = JClass("org.munin.JMXQuery")

        if _jmx_params['Password'] is not '':
            _jmx = JMXQuery(url, username, password)
        else:
            _jmx = JMXQuery(url)

        _jmx.connect()
    except:
        print "Error: ", sys.exc_info()[1]
        raise

    return _jmx

def metric_cleanup():
    '''Clean up the metric module.'''
    #shutdownJVM()

def Build_Conf():
    print "modules {\n module {\n  name = \"jmx\"\n  language = \"python\"\n }\n}\n"
    print "collection_group {\n collect_every = 30\n time_threshold = 60"
    for d in descriptors:
        print " metric {\n  name = \""+d['name']+"\"\n  title = \""+d['description']+"\"\n  value_threshold = 1.0\n }"
    print "}\n"

descriptors.append(
    Init_Metric("java_cpu_time", "java.lang:type=Threading", "CurrentThreadCpuTime", "",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))
descriptors.append(
    Init_Metric("java_cpu_user_time", "java.lang:type=Threading", "CurrentThreadUserTime", "",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))
descriptors.append(
    Init_Metric("java_memory_nonheap_committed", "java.lang:type=Memory", "NonHeapMemoryUsage", "committed",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))
descriptors.append(
    Init_Metric("java_memory_nonheap_max", "java.lang:type=Memory", "NonHeapMemoryUsage", "max",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))
descriptors.append(
    Init_Metric("java_memory_nonheap_used", "java.lang:type=Memory", "NonHeapMemoryUsage", "used",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))
descriptors.append(
    Init_Metric("java_memory_heap_committed", "java.lang:type=Memory", "HeapMemoryUsage", "committed",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))
descriptors.append(
    Init_Metric("java_memory_heap_used", "java.lang:type=Memory", "HeapMemoryUsage", "used",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))
descriptors.append(
    Init_Metric("java_memory_heap_max", "java.lang:type=Memory", "HeapMemoryUsage", "max",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))
descriptors.append(
    Init_Metric("os_memory_physical", "java.lang:type=OperatingSystem", "FreePhysicalMemorySize", "",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))
descriptors.append(
    Init_Metric("os_memory_vm", "java.lang:type=OperatingSystem", "CommittedVirtualMemorySize", "",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))
descriptors.append(
    Init_Metric("java_thread_count", "java.lang:type=Threading", "ThreadCount", "",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))
descriptors.append(
    Init_Metric("java_thread_count_peak", "java.lang:type=Threading", "PeakThreadCount", "",
        int(300), 'uint', 'both', '', '%u', Metric_Handler))

#This code is for debugging and unit testing
if __name__ == '__main__':
    try:
        if len(sys.argv) <= 1:
            debug = 1

        metric_init(_jmx_params)

        if len(sys.argv) <= 1:
            while True:
                for d in descriptors:
                    v = d['call_back'](d['name'])
                time.sleep(5)
        elif sys.argv[1] == "config":
            Build_Conf()
            metric_cleanup()

    except KeyboardInterrupt:
        print "Process interrupted."
        if _JMX_WorkerThread.running and not _JMX_WorkerThread.shuttingdown:
            _JMX_WorkerThread.shutdown()
        time.sleep(0.2)
        sys.exit(1)

You can simply run “python jmx.py config” to generate the jmx.pyconf file. Also, you’d probably need to edit the plugin to check for your own JMX values. A lot of improvement could be done to this plugin, but all the basics are there and it can give you some awesome graph to monitor your Java application.

                          

I will consider that you have a running Red5 server and you know how a Red5 application work. It’s quite easy to install, just have a look there if you need : http://osflash.org/red5/help

The Spring DTD based configuration

Here is the power of Spring, AOP and the Beans. In your Red5 installation copy the JETM jar file to the lib directory, then edit the conf/red5-common.xml as following :

<bean id="etmMonitor"
class="etm.core.monitor.NestedMonitor"
init-method="start" destroy-method="stop"/>

<bean id="etmHttpConsole"
class="etm.contrib.console.HttpConsoleServer"
init-method="start" destroy-method="stop" autowire="constructor"/>

Those two new beans are there to instantiate the JETM monitor and activate the HTTP console, by default the JETM console listen on port 40000.

So, at this point if you run your red5 server you will just see two new line in your logs :

7 juin 2009 19:48:36 etm.core.monitor.EtmMonitorSupport start
INFO: JETM 1.2.3 started.

Now, we need to define which services we want to track. For example, on the oflaDemo application, we can track what’s going on  by simply adding those other beans to this config file : oflaDemo/WEB-INF/red5-web.xml

<bean id="etmMethodCallInterceptor"
class="etm.contrib.aop.aopalliance.EtmMethodCallInterceptor"
autowire="constructor"/>

<bean id="etmAutoProxy"
class="org.springframework.aop.framework.autoproxy.BeanNameAutoProxyCreator">

<property name="interceptorNames">
<list>
<value>etmMethodCallInterceptor</value>
</list>
</property>
<property name="beanNames">
<value>web.*</value>
</property>

</bean>

From now, we are monitoring all the web.* bean from oflaDemo, this means, if you connect to oflaDemo (http://localhost:5080/demos/ofla_demo.html), view a stream etc, then go to the console (http://localhost:40000) you will see some interesting statistics like the average time spend in your Application start method etc.

JETM Console - Red5 with oflaDemo

The programmatic way

If you need more detailed statistics you can implement the lib in your application. It’s what I do now as I can have better detail and select what I really want to track, also you can manage all the JETM configuration thru an xml file. To do that you just need to change your web.handler bean definition to call an init method and implement this method in your application.

In MyTest/WEB-INF/red5-web.xml :

<bean id="web.handler"
class="org.example.red5.MyTestApplication"
init-method="init" singleton="true">

Then implement the init method in MyTestApplication :

private EtmMonitor profiler = EtmManager.getEtmMonitor();

public void init()
throws URISyntaxException
{
log.debug("Application initialized: {}", getClass().getName());

URL url = getClass().getClassLoader().getResource("jetm-mytest.xml");
try {
XmlEtmConfigurator.configure(new FileInputStream(url.getPath()));
if (!profiler.isStarted())
profiler.start();
} catch (FileNotFoundException fne) {
log.warn(fne.getMessage());
}
}

Add the jetm-mytest.xml file in your classpath like in red5/conf or in MyTest/WEB-INF/lib. The xml file look like something like that :

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE jetm-config PUBLIC "-// void.fm //DTD JETM Config 1.2//EN"
"http://jetm.void.fm/dtd/jetm_config_1_2.dtd">

<jetm-config timer="sun">

<aggregator-chain>

<chain-element class="etm.core.aggregation.BufferedTimedAggregator">
<!-- Set aggregation interval to 1 second -->
<property name="aggregationInterval">1000</property>
</chain-element>

<chain-element class="etm.contrib.aggregation.log.CommonsLoggingAggregator">
<!-- Set commons-logging log category -->
<property name="logName">etm-result</property>
</chain-element>

<chain-root class="etm.core.aggregation.persistence.PersistentRootAggregator">
<property name="aggregationInterval">10000</property>
</chain-root>

</aggregator-chain>

<extension>
<plugin class="etm.contrib.console.HttpConsoleServerPlugin">
<property name="listenPort">40000</property>
<property name="expanded">true</property>
<property name="worker-size">3</property>
</plugin>
</extension>

</jetm-config>

That’s it, now when you want to collect the statistic in one of your method or in a specific process you can just do something like that :

public void MyMethod() {

EtmPoint point = profiler.createPoint(getClass().getName()+"#MyMethod");

... Your stuff ...

point.collect();

}

All this is a succinct introduction, you can go further and do some amazing things. JETM is a really powerful tool to improve your application performance.