Skip to content
This repository was archived by the owner on Feb 24, 2021. It is now read-only.

Need to report metrics from non-master nodes as well #8

Open
damienclaveau wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Need to report metrics from non-master nodes as well #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0b21bda
Bump to ES 1.1.1
damienclaveau Apr 29, 2014
b1e3da6
Fix logger classname
damienclaveau May 11, 2014
1164b7b
Monitor non-master nodes as well, but only after a master node is ele…
damienclaveau May 11, 2014
3f4bcf6
Implement some unit tests to check the specific case of a non-master …
damienclaveau May 11, 2014
6b814a5
damienclaveau May 11, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.spinscale.elasticsearch</groupId>
<artifactId>elasticsearch-plugin-graphite</artifactId>
<version>0.2-SNAPSHOT</version>
<version>0.2.1-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Graphite monitoring plugin for Elasticsearch</description>
<url>https://github.com/spinscale/elasticsearch-graphite-plugin/</url>

<properties>
<elasticsearch.version>1.0.1</elasticsearch.version>
<elasticsearch.version>1.1.1</elasticsearch.version>
<hamcrest.version>1.3</hamcrest.version>
</properties>

Expand Down Expand Up @@ -98,4 +98,4 @@
</plugins>
</build>

</project>
</project>
44 changes: 29 additions & 15 deletions src/main/java/org/elasticsearch/service/graphite/GraphiteService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.collect.Lists;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.component.Lifecycle;
Expand Down Expand Up @@ -98,20 +99,33 @@ public GraphiteReporterThread(Pattern graphiteInclusionRegex, Pattern graphiteEx
public void run() {
while (!closed) {
DiscoveryNode node = clusterService.localNode();
boolean isClusterStarted = clusterService.lifecycleState().equals(Lifecycle.State.STARTED);

if (isClusterStarted && node != null && node.isMasterNode()) {
NodeIndicesStats nodeIndicesStats = indicesService.stats(false);
CommonStatsFlags commonStatsFlags = new CommonStatsFlags().clear();
NodeStats nodeStats = nodeService.stats(commonStatsFlags, true, true, true, true, true, true, true, true, true);
List<IndexShard> indexShards = getIndexShards(indicesService);

GraphiteReporter graphiteReporter = new GraphiteReporter(graphiteHost, graphitePort, graphitePrefix,
nodeIndicesStats, indexShards, nodeStats, graphiteInclusionRegex, graphiteExclusionRegex);
graphiteReporter.run();
} else {
if (node != null) {
logger.debug("[{}]/[{}] is not master node, not triggering update", node.getId(), node.getName());
if (node != null) {
boolean isClusterStarted = clusterService.lifecycleState().equals(Lifecycle.State.STARTED);
if (isClusterStarted) {
DiscoveryNodes nodes = clusterService.state().nodes();
// Here we are assuming that reporting metrics is useless until a Master node is elected in the Cluster
if (nodes != null && (nodes.getMasterNode() != null)) {
NodeIndicesStats nodeIndicesStats = indicesService.stats(false);
CommonStatsFlags commonStatsFlags = new CommonStatsFlags().clear();
NodeStats nodeStats = nodeService.stats(commonStatsFlags, true, true, true, true, true, true, true, true, true);
List<IndexShard> indexShards = getIndexShards(indicesService);

GraphiteReporter graphiteReporter = new GraphiteReporter(graphiteHost, graphitePort, graphitePrefix,
nodeIndicesStats, indexShards, nodeStats, graphiteInclusionRegex, graphiteExclusionRegex);
graphiteReporter.run();
} else {
if (logger.isDebugEnabled()) {
logger.debug("[{}]/[{}] not triggering update, Cluster has no Master node, ", node.getId(), node.getName());
}
}
} else {
if (logger.isDebugEnabled()) {
logger.debug("[{}]/[{}] not triggering update, Cluster is not started", node.getId(), node.getName());
}
}
} else {
if (logger.isDebugEnabled()) {
logger.debug("Not triggering update, Node is Null");
}
}

Expand All @@ -135,4 +149,4 @@ private List<IndexShard> getIndexShards(IndicesService indicesService) {
return indexShards;
}
}
}
}
76 changes: 61 additions & 15 deletions src/test/java/org/elasticsearch/module/graphite/test/GraphitePluginIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,15 @@

public class GraphitePluginIntegrationTest {

public static final String GRAPHITE_SERVER_HOST = "localhost";
public static final int GRAPHITE_SERVER_PORT = 12345;

private GraphiteMockServer graphiteMockServer;

private String clusterName = UUID.randomUUID().toString().replaceAll("-", "");
private String index = UUID.randomUUID().toString().replaceAll("-", "");
private String type = UUID.randomUUID().toString().replaceAll("-", "");
private String defaultPrefix = "elasticsearch." + clusterName;
private Node node;

@Before
Expand All @@ -43,72 +45,116 @@ public void stopGraphiteServer() throws Exception {

@Test
public void testThatIndexingResultsInMonitoring() throws Exception {
node = createNode(clusterName, GRAPHITE_SERVER_PORT, "1s");
node = createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null);
IndexResponse indexResponse = indexElement(node, index, type, "value");
assertThat(indexResponse.getId(), is(notNullValue()));

Thread.sleep(2000);

ensureValidKeyNames();
assertGraphiteMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing._all.indexCount 1");
assertGraphiteMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing." + type + ".indexCount 1");
assertGraphiteMetricIsContained("elasticsearch." + clusterName + ".node.jvm.threads.peakCount ");
assertGraphiteMetricIsContained(defaultPrefix + ".indexes." + index + ".id.0.indexing._all.indexCount 1");
assertGraphiteMetricIsContained(defaultPrefix + ".indexes." + index + ".id.0.indexing." + type + ".indexCount 1");
assertGraphiteMetricIsContained(defaultPrefix + ".node.jvm.threads.peakCount ");
}


@Test
public void testThatFieldExclusionWorks() throws Exception {
String excludeRegex = ".*\\.peakCount";
node = createNode(clusterName, GRAPHITE_SERVER_PORT, "1s", null, excludeRegex);
node = createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null, null, excludeRegex);

IndexResponse indexResponse = indexElement(node, index, type, "value");
assertThat(indexResponse.getId(), is(notNullValue()));

Thread.sleep(2000);

ensureValidKeyNames();
assertGraphiteMetricIsNotContained("elasticsearch." + clusterName + ".node.jvm.threads.peakCount ");
assertGraphiteMetricIsNotContained(defaultPrefix + ".node.jvm.threads.peakCount ");
}

@Test
public void testThatFieldInclusionWinsOverExclusion() throws Exception {
String excludeRegex = ".*" + clusterName + ".*";
String includeRegex = ".*\\.peakCount";
node = createNode(clusterName, GRAPHITE_SERVER_PORT, "1s", includeRegex, excludeRegex);
node = createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null, includeRegex, excludeRegex);

IndexResponse indexResponse = indexElement(node, index, type, "value");
assertThat(indexResponse.getId(), is(notNullValue()));

Thread.sleep(2000);

ensureValidKeyNames();
assertGraphiteMetricIsNotContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing._all.indexCount 1");
assertGraphiteMetricIsContained("elasticsearch." + clusterName + ".node.jvm.threads.peakCount ");
assertGraphiteMetricIsNotContained(defaultPrefix + ".indexes." + index + ".id.0.indexing._all.indexCount 1");
assertGraphiteMetricIsContained(defaultPrefix + ".node.jvm.threads.peakCount ");
}

@Test(expected = ProvisionException.class)
public void testThatBrokenRegexLeadsToException() throws Exception {
String excludeRegex = "*.peakCount";
createNode(clusterName, GRAPHITE_SERVER_PORT, "1s", null, excludeRegex);
createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null, null, excludeRegex);
}


@Test
public void masterFailOverShouldWork() throws Exception {
node = createNode(clusterName, GRAPHITE_SERVER_PORT, "1s");
// Generate a new cluster not to interfer with the precedents tests
String clusterName = UUID.randomUUID().toString().replaceAll("-", "");
String defaultPrefix = "elasticsearch." + clusterName;

node = createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null);
IndexResponse indexResponse = indexElement(node, index, type, "value");
assertThat(indexResponse.getId(), is(notNullValue()));

Node origNode = node;
node = createNode(clusterName, GRAPHITE_SERVER_PORT, "1s");
node = createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null);
graphiteMockServer.content.clear();
origNode.stop();
indexResponse = indexElement(node, index, type, "value");
assertThat(indexResponse.getId(), is(notNullValue()));

// wait for master fail over and writing to graph reporter
Thread.sleep(2000);
assertGraphiteMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing._all.indexCount 1");
assertGraphiteMetricIsContained(defaultPrefix + ".indexes." + index + ".id.0.indexing._all.indexCount 1");
}

@Test
public void testThatNotMasterNodeIsMonitoredAsWell() throws Exception {
// Generate a new cluster not to interfer with the precedents tests
String clusterName = UUID.randomUUID().toString().replaceAll("-", "");
String defaultPrefix = "elasticsearch." + clusterName;

// create a master node without the plugin disabled
Node masterNode = createNode(clusterName, true);
graphiteMockServer.content.clear();
Thread.sleep(2000);
// create a non-master node with the plugin enabled,
// and a specific prefix to be sure that the metrics are emitted by that node
String specificPrefix = defaultPrefix + ".dataNode";
node = createNode(clusterName, false, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", specificPrefix);
Thread.sleep(2000);

assertGraphiteMetricIsContained(specificPrefix + ".node.jvm.uptime ");
}

@Test
public void testThatNotMasterNodeIsNotMonitoredWhenNoMasterIsFound() throws Exception {
// Generate a new cluster not to interfer with the precedents tests
String clusterName = UUID.randomUUID().toString().replaceAll("-", "");
String defaultPrefix = "elasticsearch." + clusterName;

// create a non-master node with the plugin enabled,
// and no metric should be reported until a master is elected
String specificPrefix = defaultPrefix + ".dataNode";
graphiteMockServer.content.clear();
node = createNode(clusterName, false, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", specificPrefix);
Thread.sleep(4000);
assertGraphiteMetricIsNotContained(specificPrefix + ".node.jvm.uptime ");

// create a master node to be joigned...
graphiteMockServer.content.clear();
Node masterNode = createNode(clusterName, true);
Thread.sleep(4000);
//the dataNode should now start reporting metrics
assertGraphiteMetricIsContained(specificPrefix + ".node.jvm.uptime ");
}

// the stupid hamcrest matchers have compile erros depending whether they run on java6 or java7, so I rolled my own version
Expand All @@ -135,4 +181,4 @@ private IndexResponse indexElement(Node node, String index, String type, String
setSource("field", fieldValue)
.execute().actionGet();
}
}
}
28 changes: 20 additions & 8 deletions src/test/java/org/elasticsearch/module/graphite/test/NodeTestHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,34 @@

public class NodeTestHelper {

public static Node createNode(String clusterName, int graphitePort, String refreshInterval) throws IOException {
return createNode(clusterName, graphitePort, refreshInterval, null, null);
public static Node createNode(String clusterName, boolean nodeMaster) throws IOException {
return createNode(clusterName, nodeMaster, null, 0, null, null, null, null);
}

public static Node createNode(String clusterName, int graphitePort, String refreshInterval, String includeRegex,
public static Node createNode(String clusterName, boolean nodeMaster, String graphiteHost, int graphitePort, String refreshInterval, String graphitePrefix) throws IOException {
return createNode(clusterName, nodeMaster, graphiteHost, graphitePort, refreshInterval, graphitePrefix, null, null);
}

public static Node createNode(String clusterName, boolean nodeMaster, String graphiteHost, int graphitePort, String refreshInterval, String graphitePrefix, String includeRegex,
String excludeRegex) throws IOException {
ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder();

settingsBuilder.put("gateway.type", "none");
settingsBuilder.put("cluster.name", clusterName);
settingsBuilder.put("node.master", nodeMaster);
settingsBuilder.put("index.number_of_shards", 1);
settingsBuilder.put("index.number_of_replicas", 1);

settingsBuilder.put("metrics.graphite.host", "localhost");
settingsBuilder.put("metrics.graphite.port", graphitePort);
settingsBuilder.put("metrics.graphite.every", refreshInterval);
settingsBuilder.put("discovery.zen.ping.multicast.enabled", false);
settingsBuilder.put("discovery.zen.ping.unicast.hosts", "localhost:9300,localhost:9301,localhost:9302");

if (Strings.hasLength(graphiteHost)) {
settingsBuilder.put("metrics.graphite.host", graphiteHost);
settingsBuilder.put("metrics.graphite.port", graphitePort);
settingsBuilder.put("metrics.graphite.every", refreshInterval);
if (Strings.hasLength(graphitePrefix)) {
settingsBuilder.put("metrics.graphite.prefix", graphitePrefix);
}
}

if (Strings.hasLength(includeRegex)) {
settingsBuilder.put("metrics.graphite.include", includeRegex);
Expand All @@ -39,4 +51,4 @@ public static Node createNode(String clusterName, int graphitePort, String refre

return NodeBuilder.nodeBuilder().settings(settingsBuilder.build()).node();
}
}
}
4 changes: 2 additions & 2 deletions src/test/resources/logging.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
rootLogger: INFO, console

logger:
service.graphite: DEBUG
org.elasticsearch.service.graphite: DEBUG
plugin.graphite.GraphitePlugin: DEBUG

appender:
console:
type: console
layout:
type: consolePattern
conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"