WebLogic Diagnostics

The WebLogic Diagnostic Framework (WLDF) is a collection of services for monitoring the runtime
behavior of applications, such as

Logging
Instrumentation - apply instrumentation monitors to arbitrary points in application code.
Harvester - schedule tasks to record metric data from a configurable set of JMX MBeans.
Watches and Notifications - WLDF watch rules can be used to monitor log messages,
instrumentation events, or harvested metric data.

The WLDF services are designed to complement each other. To get a feeling of the different services,
we use an example application. It consists of a stateless session bean, a message driven bean and a
test servlet. The session bean and message driven bean are presented below; the servlet is
presented later on in this article. The stateless session bean:
package datamodel.logic;
import datamodel.entities.Bestelling;
import datamodel.entities.Klant;
import java.util.List;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import javax.ejb.Stateless;
import
import
import
import
import
import
import
import
import
import

javax.jms.Connection;
javax.jms.ConnectionFactory;
javax.jms.JMSException;
javax.jms.MessageProducer;
javax.jms.ObjectMessage;
javax.jms.Queue;
javax.jms.Session;
javax.naming.Context;
javax.naming.InitialContext;
javax.naming.NamingException;

@Stateless(name = "Bedrijf", mappedName = "ejb/Bedrijf")

public class BedrijfBean implements Bedrijf {
//@Resource(name = "jms/ConnectionFactory", type = ConnectionFactory.class)
private ConnectionFactory connectionFactory;
//@Resource(name = "jms/Queue", type = Queue.class)
private Queue queue;
private Connection connection;
private Session session;
private MessageProducer messageProducer;
public BedrijfBean() {
}
@PostConstruct
public void init() {
try {
Context context = new InitialContext();
connectionFactory = (ConnectionFactory)context.lookup("jms/ConnectionFactory");
queue = (Queue)context.lookup("jms/Queue");
} catch (NamingException e) {
e.printStackTrace();
}
try {
connection = connectionFactory.createConnection();

session = connection.createSession(false, Session.CLIENT_ACKNOWLEDGE);

messageProducer = session.createProducer(queue);
} catch (JMSException e) {
e.printStackTrace();
}
}
@PreDestroy
public void release() {
try {
if (connection != null) {
connection.close();
}
} catch (JMSException e) {
e.printStackTrace();
}
}
public Klant addKlant(Klant klant) {
System.out.println(klant);
sendMessage(klant);
return null;
}
public void removeKlant(Integer klantnummer) {
System.out.println(klantnummer);
}
public Bestelling addBestelling(Bestelling bestelling) {
System.out.println(bestelling);
return null;
}
public void updateBestelling(Bestelling bestelling) {
System.out.println(bestelling);
}
public Bestelling getBestelling(Integer bestellingnummer) {
System.out.println(bestellingnummer);
return null;
}
public List<Klant> getKlanten() {
System.out.println("nothing");
return null;
}
private void sendMessage(Klant klantToSend) {
try {
ObjectMessage message = session.createObjectMessage(klantToSend);
messageProducer.send(message);
} catch (JMSException e) {
e.printStackTrace();
}
}
}
The message driven bean:
package datamodel.logic;
import datamodel.entities.Klant;
import javax.ejb.ActivationConfigProperty;
import javax.ejb.MessageDriven;
import
import
import
import

javax.jms.JMSException;
javax.jms.Message;
javax.jms.MessageListener;
javax.jms.ObjectMessage;

@MessageDriven(mappedName = "jms/Queue", activationConfig = {

@ActivationConfigProperty(propertyName = "destinationName", propertyValue =
"jms/Queue"),
@ActivationConfigProperty(propertyName = "destinationType", propertyValue =

"javax.jms.Queue")
})
public class BedrijfMDB implements MessageListener {
public BedrijfMDB() {
}
public void onMessage(Message message) {
ObjectMessage objectMessage = (ObjectMessage) message;
Klant klant = null;
try {
klant = (Klant) objectMessage.getObject();
message.acknowledge();
} catch (JMSException e) {
e.printStackTrace();
}
System.out.println("MESSAGING KLANT " + klant);
}
}

Instrumentation
Say we are interested in the performance of adding a customer. In particular, we want to know how
long it takes the session bean to produce a message and send to a JMS queue, further we are
interested in how long the processing of the message takes by the message driven bean. To add
instrumentation, we create an application-specific diagnostic module, for example,
<?xml version="1.0" encoding="UTF-8"?>
<wldf-resource xmlns="http://xmlns.oracle.com/weblogic/weblogic-diagnostics"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://xmlns.oracle.com/weblogic/weblogic-diagnostics
http://xmlns.oracle.com/weblogic/weblogic-diagnostics/1.0/weblogicdiagnostics.xsd">
<instrumentation>
<enabled>true</enabled>
<wldf-instrumentation-monitor>
<name>AddKlantInstrumentationEJB</name>
<action>MethodInvocationStatisticsAction</action>
<location-type>around</location-type>
<pointcut>execution(* datamodel.logic.BedrijfBean addKlant(...))</pointcut>
</wldf-instrumentation-monitor>
<wldf-instrumentation-monitor>
<name>OnMessageInstrumentationMDB</name>
<action>MethodInvocationStatisticsAction</action>
<location-type>around</location-type>
<pointcut>execution(* datamodel.logic.BedrijfMDB onMessage(...))</pointcut>
</wldf-instrumentation-monitor>
</instrumentation>
</wldf-resource>

This should be named weblogic-diagnostics.xml, and packaged in the application archive below the
META-INF directory.
The module defines two monitors. The code instrumented by each monitor is determined by a
pointcut expression. Each monitor generates an event whenever the instrumented code is called. The
monitored points are specified using declarative rules (pointcuts), and implemented using byte code
modification. The events that are captured include a diagnostic context identifier, which allows a
stream of related events to be correlated whether they occur within a server, or across JMS
messages, remote RMI, or SOAP calls.
In addition to the instrumentation point, each monitor configures an action. There are several types of
actions, such as record elapsed time, take a stack or thread dump, or record the number of times the
monitored code has been called. Three types of monitors can be defined: before, after or around.
Note that the MethodInvocationStatisticsAction aggregates information for each monitor and stores it
in an in-memory JMX MBean that can be queried.

A diagnostic system module is also required. Diagnostic application modules only contain application
scoped monitors. System modules contain system-scoped monitors and other WLDF configuration
data, such as that for the harvester, watches, and notifications. Note that instrumentation must be
enabled in a system module that is targeted to each server. This allows it to be easily switched on or
off. Each server can have at most one targeted diagnostic system module. To create a diagnostic
system module use the WebLogic Console. We simply need to target the module to the appropriate
server and enable instrumentation using the Enabled checkbox on the Instrumentation Configuration
tab.
The information recorded by the MethodInvocationStatisticsAction is available through the
application's WLDFInstrumentationRuntime MBean. This MBean has a MethodInvocationStatistics
attribute that returns a nested set of maps containing of all the recorded statistics for the application. It
also has an getMethodInvocationStatisticsData() operation that allows you to query for a subset of the
data. The example WLST script queries the WLDFInstrumentationRuntimeMBean and displays the
results
connect('username','password');
cd('serverRuntime:/WLDFRuntime/WLDFRuntime/WLDFInstrumentationRuntimes/<application-name>');
def formatTime(t):
return "%.2f" % (t/1e6)
print('OBTAINING STATISTICS');
for classStats in cmo.getMethodInvocationStatistics().entrySet():
print('OBTAINED THE CLASS STATISTICS');
for methodStats in classStats.value.entrySet():
print('OBTAINED THE METHOD STATISTICS');
for signatureStats in methodStats.value.entrySet():
print('PRINTING SOME OUTPUT');
print "%s.%s(): %d %s %s %s %s" % (
classStats.key,
methodStats.key,
signatureStats.value['count'],
formatTime(signatureStats.value['min']),
formatTime(signatureStats.value['avg']),
formatTime(signatureStats.value['max']),
formatTime(signatureStats.value['std_deviation']),)

Here are some results

datamodel.logic.BedrijfBean.addKlant(): 50 1.52 8.46 66.94 11.09
datamodel.logic.BedrijfMDB onMessage(): 50 0.13 0.75 2.46 0.70
The values are the number of the times the method has been called, followed by the minimum,
average, and maximum times, and the standard deviation. All the times are in milliseconds. These
numbers were obtained by making a few requests to the application using a browser. If we applied a
more realistic work load using a performance testing tool such as The Grinder, we could get a very
good feel for the cost.
The Grinder is a framework for running test scripts. It consists of a console (coordinates processes,
collates and displays statistics) an agent which manages worker processes and worker processes
which interpret test scripts and perform tests using a number of worker threads. To start the
environment we first create a few start scripts:
setGrinderEnv.sh
#!/bin/sh
GRINDERPATH=/home/oracle/grinder/grinder-3.4
export GRINDERPATH
GRINDERPROPERTIES=${GRINDERPATH}/examples/grinder.properties
export GRINDERPROPERTIES
CLASSPATH=${GRINDERPATH}/lib/grinder.jar:${CLASSPATH}
export CLASSPATH
JAVA_HOME=/home/oracle/bea/jrrt-4.0.1-1.6.0
export JAVA_HOME

PATH=${JAVA_HOME}/bin:${PATH}
export PATH
# Memory arguments for optimizing the throughput using JRockit
USER_MEM_ARGS="-Xms512m -Xmx512m -Xss128k -Xgcprio:throughput"
export USER_MEM_ARGS
startConsole.sh
#!/bin/sh
. /home/oracle/grinder/grinder-3.4/setGrinderEnv.sh
java ${USER_MEM_ARGS} -cp ${CLASSPATH} net.grinder.Console
startAgent.sh
#!/bin/sh
. /home/oracle/grinder/grinder-3.4/setGrinderEnv.sh
java ${USER_MEM_ARGS} -cp ${CLASSPATH} net.grinder.Grinder ${GRINDERPROPERTIES}

An example grinder.properties file looks as follows:

# The file name of the script to run.
# Relative paths are evaluated from the directory containing the properties file.
# The default is "grinder.py".
grinder.script = http.py
# The number of worker processes each agent should start.
# The default is 1.
grinder.processes = 1
# The number of worker threads each worker process should start.
# The default is 1.
grinder.threads = 1
# The number of runs each worker process will perform.
# When using the console this is usually set to 0, meaning
# "run until the console sends a stop or reset signal".
# The default is 1.
grinder.runs = 0

The Grinder uses a python script to define which test to run, the http.py script defined in the properties
looks as follows:
# An example using the HTTP plugin that shows the retrieval of a single page via HTTP.
from net.grinder.script.Grinder import grinder
from net.grinder.script import Test
from net.grinder.plugin.http import HTTPRequest
test1 = Test(1, "Request resource")
request1 = test1.wrap(HTTPRequest())
class TestRunner:
def __call__(self):
result = request1.GET("http://localhost:7001/Bedrijf/testservlet")

The servlet (testservlet) looks as follows

package userinterface.servlets.test;
import datamodel.entities.Bestelling;
import datamodel.entities.Klant;
import datamodel.exceptions.BedrijfException;
import datamodel.logic.Bedrijf;
import java.io.IOException;
import java.util.Date;

import java.util.List;
import javax.naming.Context;
import javax.naming.InitialContext;
import javax.naming.NamingException;
import
import
import
import

javax.servlet.ServletException;
javax.servlet.http.HttpServlet;
javax.servlet.http.HttpServletRequest;
javax.servlet.http.HttpServletResponse;

public class TestServlet extends HttpServlet {

public void doGet(HttpServletRequest request, HttpServletResponse response) throws
ServletException, IOException {
Bedrijf bedrijf = getBedrijf();
Bestelling bestelling = bedrijf.getBestelling(610);
System.out.println(bestelling);
try {
bedrijf.addKlant(createKlant());
} catch (BedrijfException e) {
System.out.println(e.getMessage());
}
try {
bedrijf.removeKlant(123);
} catch (BedrijfException e) {
System.out.println(e.getMessage());
}
try {
bedrijf.addBestelling(createBestelling());
} catch (BedrijfException e) {
System.out.println(e.getMessage());
}
if (bestelling != null) {
bestelling.setVerstuurDatum(new Date());
bestelling.setCommissionPlan("B");
bedrijf.updateBestelling(bestelling);
}
List<Klant> klanten = bedrijf.getKlanten();
if (klanten != null) {
for (Klant klant: klanten) {
System.out.println(klant);
for (Bestelling klantbestelling: klant.getBestellingen()) {
System.out.println(" - " + klantbestelling);
}
}
}
}
private Klant createKlant() {
Klant klant = new Klant();
klant.setKlantnummer(123);
klant.setNaam("Ikke");
klant.setAdres("Ergens");
klant.setStad("Hier");
klant.setProvincie("ZH");
klant.setPostcode("4152HU");
klant.setGebied(1);
klant.setTelefoonnummer("09-92839");
klant.setReputatieNummer(2);
klant.setKredietlimiet(100.0);
klant.setCommentaar("Blablabla");
return klant;
}
private Bestelling createBestelling() {
Bestelling bestelling = new Bestelling();
bestelling.setBestellingnummer(600);
bestelling.setBestellingDatum(new Date());
bestelling.setCommissionPlan("Z");
bestelling.setKlantnummer(100);

return bestelling;
}
private Bedrijf getBedrijf() {
Context context = null;
Bedrijf bedrijf = null;
try {
context = new InitialContext();
bedrijf = (Bedrijf)context.lookup("ejb/Bedrijf#datamodel.logic.Bedrijf");
} catch (NamingException e) {
e.printStackTrace();
}
return bedrijf;
}
}

To start the test run the startConsole script and subsequently run the startAgent script (use the
grinder console to send a start event to the agent). If we let the test run for a while and then run the
WLST script to get the instrumentation results, we have:
datamodel.logic.BedrijfBean.addKlant(): 33994 0.28 1.44 2196.76 12.76
datamodel.logic.BedrijfMDB onMessage(): 33994 0.07 0.82 2197.10 12.32

Harvesting
A harvester is configured in a diagnostics system module. By using the WebLogic Console we can
access the diagnostic system module's Collected Metrics Configuration tab. Here we see that we can
specify the sampling period, which defaults to 5 minutes. On the page we can also add harvesting
rules to collect data from built-in MBeans. For example, we might want to record the number of JDBC
connections in use. By using the WebLogic Console we define a new harvesting metric that collects
the value of the ActiveConnectionsCurrentCount attribute of the JDBCDataSourceRuntime MBean by
selecting the MBean and the attribute from the provided lists. To configure a harvester:

Click Services, Diagnostics, Diagnostic Modules and click New.

Click the created module, Configuration, Collected Metrics and click New.
MBean Server: ServerRuntime, click Next.
Select an MBean: JDBCDataSourceRuntimeMBean, click Next.
Collected Attributes: Select ActiveConnectionsCurrentCount, click Next.
Collected Instances: Select some servers, click Finish.

We can also capture the statistics from the application WLDFInstrumentationRuntime MBean. In this
case, we have to provide an attribute expression in a particular form, for example,
MethodInvocationStatistics(*)(*)(*)(count|avg|min|max). This expression harvests the count, average,
minimum, and maximum statistics from monitors that use the MethodInvocationStatisticsAction.
The listing below shows the configuration file for the diagnostic system module having configured it to
harvest from the JDBCDataSourceRuntime and WLDFInstrumentationRuntime MBeans.
<wldf-resource ...>
<name>MySystemModule</name>
<instrumentation>
<enabled>true</enabled>
</instrumentation>
<harvester>
<harvested-type>
<name>weblogic.management.runtime.JDBCDataSourceRuntimeMBean</name>
<harvested-attribute>ActiveConnectionsCurrentCount</harvestedattribute>
<harvestedinstance>com.bea:Name=VideotheekDataSource,ServerRuntime=AdminServer,Type=JDBCDataSourceRuntim
e</harvested-instance>
<namespace>ServerRuntime</namespace>
</harvested-type>

<harvested-type>
<name>weblogic.management.runtime.WLDFInstrumentationRuntimeMBean</name>
<harvestedattribute>MethodInvocationStatistics(*)(*)(*)(avg,count,min,max)</harvested-attribute>
</harvested-type>
</harvester>
</wldf-resource>

The servers to which the diagnostic module is targeted are now recording information in their
diagnostic archives. By using the WLDF console extension, we can see the information captured from
the JDBCDataSourceRuntime MBean. To enable the WLDF console extension, copy the file
diagnostics-console-extension.jar located in the directory ${WL_HOME}/server/lib/console-ext, and
copy this file to the directory ${DOMAIN_HOME}/console-ext. Click in the admin console on
Preferences, select the Extensions tab and enable diagnostics-console-extension. Restart the admin
server.
Due to its complex format, the WLDF console extension cannot display the data from the
WLDFInstrumentationRuntime MBean. To get at the data, we have to export it from the archive, for
example, by using the following WLST commands,
# connect to the appropriate server
connect('username','password', 't3://hostname:port');
# use the exportDiagnosticDataFromServer command to export the data
# valid values for logicalName are: HarvestedDataArchive, EventsDataArchive, ServerLog,
DomainLog, HTTPAccessLog, WebAppLog, ConnectorLog, and JMSMessageLog.
exportDiagnosticDataFromServer(logicalName="HarvestedDataArchive",
exportFileName="export.xml");

The data is exported in an XML format that represents a set of tabular rows.
Each WebLogic server has a persistent diagnostic archive that is used to store captured data. The
archive can be persisted using either a file-based or JDBC-based persistent store. Be wary of
instrumenting too many methods, harvesting too many metrics, or using a very short sampling period.
This will quickly fill up the archive, and the overhead of the instrumentation may become measurable.
The data in the archive obviously cannot be allowed to grow indefinitely. WebLogic Server allows data
retirement policies to be set for a each class of data in the diagnostic archive that regularly deletes
data of a certain age. Data retirement policies can be used for both file- and database-based stores.
For file stores, a preferred maximum size can also be configured - WebLogic Server will regularly
remove old records to keep the file below the configured size.

Watching and Notifying

A watch is a rule that is used to monitor log records, WLDF events, and harvest metric data for some
situations. When a watch fires, it triggers one or more notifications. A notification allows users and
other systems to be informed of the situation using JMX, SNMP, JMS, or email. Both watches and
notifications are configured in a diagnostic system module using the WebLogic Console. We could, for
example, configure a watch rule that fires whenever the harvester discovers there is less than
10MByte of free heap space memory available. Watches can have an associated alarm setting. This
prevents the same watch from firing within either a configured period (an automatic reset alarm), or
until an administrator resets the alarm (a manual reset alarm).
Configure a harvester:

Click Services, Diagnostics, Diagnostic Modules and click New.

Click the created module, Configuration, Collected Metrics and click New.
MBean Server: ServerRuntime, click Next.
Select an MBean: JVMRuntimeMBean, click Next.
Collected Attributes: Select HeapFreeCurrent, click Next.
Collected Instances: Select some servers, click Finish.

Configure the Watch and Notification

Click the tab Watches and Notifications, Notifications and click New.
Select as Type: JMS Message and click Next.
Enter a Notification Name, for example JVMNotification and click Next.
Enter the JNDI's of the destination and the connection factory and click Finish.
Click the tab Watches and click New.
Enter a Watch Name and select as Watch Type: Collected Metrics, click Next.
Click Add Expressions, MBean Server: ServerRuntime, Next, MBean Type:
JVMRuntimeMBean, Next, Select the same server instance as the chosen in the harvester,
Next, select Message Attribute: HeapFreeCurrent, select operator: <, enter as value:
10000000, click Finish.
Click Next and click Use an automatic reset alarm, for example of 60 seconds.
Click Next and select the configured notification.
Click Finish.

The following module is created:

<wldf-resource ...>
<name>JVMModule</name>
<harvester>
<harvested-type>
<name>weblogic.management.runtime.JVMRuntimeMBean</name>
<harvested-attribute>HeapFreeCurrent</harvested-attribute>
<harvestedinstance>com.bea:Name=AdminServer,ServerRuntime=AdminServer,Type=JVMRuntime</harvestedinstance>
<namespace>ServerRuntime</namespace>
</harvested-type>
</harvester>
<watch-notification>
<watch>
<name>JVMWatch</name>
<enabled>true</enabled>
<rule-type>Harvester</rule-type>
<ruleexpression>(${ServerRuntime//[weblogic.management.runtime.JVMRuntimeMBean]com.bea:Name=AdminSe
rver,ServerRuntime=AdminServer,Type=JVMRuntime//HeapFreeCurrent} < 10000000)</rule-expression>
<alarm-type>AutomaticReset</alarm-type>
<alarm-reset-period>60000</alarm-reset-period>
<notification>JVMNotification</notification>
</watch>
<jms-notification>
<name>JVMNotification</name>
<enabled>true</enabled>
<destination-jndi-name>jms/Queue</destination-jndi-name>
<connection-factory-jndi-name>jms/ConnectionFactory</connectionfactory-jndi-name>
</jms-notification>
</watch-notification>
</wldf-resource>

Monitoring using WLST

JMX MBean information can also be queried by using WLST, for example to obtain data concerning
the used datasource we can use
servers=domainRuntimeService.getServerRuntimes();
if (len(servers) > 0):
for server in servers:
jdbcServiceRT = server.getJDBCServiceRuntime();
dataSources = jdbcServiceRT.getJDBCDataSourceRuntimeMBeans();
if (len(dataSources) > 0):
for dataSource in dataSources:
print(dataSource.getWaitingForConnectionHighCount());
print(dataSource.getActiveConnectionsHighCount());

or to query our JMS environment

servers = domainRuntimeService.getServerRuntimes();
if (len(servers) > 0):
for server in servers:
jmsRuntime = server.getJMSRuntime();
jmsServers = jmsRuntime.getJMSServers();
for jmsServer in jmsServers:
destinations = jmsServer.getDestinations();
for destination in destinations:
# print some runtime info
print(destination.getMessagesHighCount());
print(destination.getConsumersHighCount());
# do some actions
selector = "JMSMessageID like '%45%'";
timeout = 0;
destination.getMessages(selector,timeout);
destination.deleteMessages(selector);

Last but not least we can also create scripts that perform checks on WebLogic servers and deployed
applications, for example
class DeploymentInfo:
def __init__(self,name,target):
self.name = name;
self.target = target;
def getName(self):
return self.name;
def getTarget(self):
return self.target;
print 'CONNECT TO ADMIN SERVER';
connect('username', 'password');
print 'OBTAINING DEPLOYMENT INFORMATION';
deploymentsInfo = [];
applications = cmo.getAppDeployments();
for application in applications:
name = application.getName();
target = application.getTargets()[0].getName();
deploymentsInfo.append(DeploymentInfo(name, target));
libraries = cmo.getLibraries();
for library in libraries:
name = library.getName();
target = library.getTargets()[0].getName();
deploymentsInfo.append(DeploymentInfo(name, target));
print 'CHANGE TO DOMAIN RUNTIME ENVIRONMENT';
domainRuntime();
print 'SERVER LIFE CYCLE INFORMATION';
serverLifeCycles = cmo.getServerLifeCycleRuntimes();
for serverLifeCycle in serverLifeCycles:
print 'Server: ' + serverLifeCycle.getName() + ', State: ' +
serverLifeCycle.getState();
if (serverLifeCycle.getState() == 'RUNNING'):
cd('ServerRuntimes/' + serverLifeCycle.getName());
jvm = cmo.getJVMRuntime();
print 'Java VM Vendor: ' + jvm.getJavaVMVendor();
print 'Heap Free Space: ' + repr(jvm.getHeapFreePercent()) + '%';
print 'Heap Size Current: ' + repr(jvm.getHeapSizeCurrent());
print 'Maximum Heap Size: ' + repr(jvm.getHeapSizeMax());
cd('/');
if (serverLifeCycle.getState() != 'RUNNING'):
start(serverLifeCycle.getName(), 'Server');
print 'APPLICATION LIFE CYCLE INFORMATION';
applicationRuntime = cmo.getAppRuntimeStateRuntime();
for deploymentInfo in deploymentsInfo:

state = applicationRuntime.getCurrentState(deploymentInfo.getName(),
deploymentInfo.getTarget())
print 'Application: ' + deploymentInfo.getName() + ', State: ' + state;
if (state != 'STATE_ACTIVE'):
startApplication(deploymentInfo.getName());

The script prints information about the heap usage of the JVM. It also checks if the servers are still
running and if they are not the script starts them. The same is done for all the deployed applications
and libraries on the WebLogic servers.

WebLogic rules!