diff --git a/pom.xml b/pom.xml index 9f1a21e..36f8f83 100644 --- a/pom.xml +++ b/pom.xml @@ -5,13 +5,13 @@ 4.0.0 de.spinscale.elasticsearch elasticsearch-plugin-graphite - 0.2-SNAPSHOT + 0.2.1-SNAPSHOT jar Graphite monitoring plugin for Elasticsearch https://github.com/spinscale/elasticsearch-graphite-plugin/ - 1.0.1 + 1.1.1 1.3 @@ -98,4 +98,4 @@ - + \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/service/graphite/GraphiteService.java b/src/main/java/org/elasticsearch/service/graphite/GraphiteService.java index ea8be25..705481b 100644 --- a/src/main/java/org/elasticsearch/service/graphite/GraphiteService.java +++ b/src/main/java/org/elasticsearch/service/graphite/GraphiteService.java @@ -5,6 +5,7 @@ import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.Lifecycle; @@ -98,20 +99,33 @@ public GraphiteReporterThread(Pattern graphiteInclusionRegex, Pattern graphiteEx public void run() { while (!closed) { DiscoveryNode node = clusterService.localNode(); - boolean isClusterStarted = clusterService.lifecycleState().equals(Lifecycle.State.STARTED); - - if (isClusterStarted && node != null && node.isMasterNode()) { - NodeIndicesStats nodeIndicesStats = indicesService.stats(false); - CommonStatsFlags commonStatsFlags = new CommonStatsFlags().clear(); - NodeStats nodeStats = nodeService.stats(commonStatsFlags, true, true, true, true, true, true, true, true, true); - List indexShards = getIndexShards(indicesService); - - GraphiteReporter graphiteReporter = new GraphiteReporter(graphiteHost, graphitePort, graphitePrefix, - nodeIndicesStats, indexShards, nodeStats, graphiteInclusionRegex, graphiteExclusionRegex); - graphiteReporter.run(); - } else { - if (node != null) { - logger.debug("[{}]/[{}] is not master node, not triggering update", node.getId(), node.getName()); + if (node != null) { + boolean isClusterStarted = clusterService.lifecycleState().equals(Lifecycle.State.STARTED); + if (isClusterStarted) { + DiscoveryNodes nodes = clusterService.state().nodes(); + // Here we are assuming that reporting metrics is useless until a Master node is elected in the Cluster + if (nodes != null && (nodes.getMasterNode() != null)) { + NodeIndicesStats nodeIndicesStats = indicesService.stats(false); + CommonStatsFlags commonStatsFlags = new CommonStatsFlags().clear(); + NodeStats nodeStats = nodeService.stats(commonStatsFlags, true, true, true, true, true, true, true, true, true); + List indexShards = getIndexShards(indicesService); + + GraphiteReporter graphiteReporter = new GraphiteReporter(graphiteHost, graphitePort, graphitePrefix, + nodeIndicesStats, indexShards, nodeStats, graphiteInclusionRegex, graphiteExclusionRegex); + graphiteReporter.run(); + } else { + if (logger.isDebugEnabled()) { + logger.debug("[{}]/[{}] not triggering update, Cluster has no Master node, ", node.getId(), node.getName()); + } + } + } else { + if (logger.isDebugEnabled()) { + logger.debug("[{}]/[{}] not triggering update, Cluster is not started", node.getId(), node.getName()); + } + } + } else { + if (logger.isDebugEnabled()) { + logger.debug("Not triggering update, Node is Null"); } } @@ -135,4 +149,4 @@ private List getIndexShards(IndicesService indicesService) { return indexShards; } } -} +} \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/module/graphite/test/GraphitePluginIntegrationTest.java b/src/test/java/org/elasticsearch/module/graphite/test/GraphitePluginIntegrationTest.java index 2051240..3142f55 100644 --- a/src/test/java/org/elasticsearch/module/graphite/test/GraphitePluginIntegrationTest.java +++ b/src/test/java/org/elasticsearch/module/graphite/test/GraphitePluginIntegrationTest.java @@ -18,6 +18,7 @@ public class GraphitePluginIntegrationTest { + public static final String GRAPHITE_SERVER_HOST = "localhost"; public static final int GRAPHITE_SERVER_PORT = 12345; private GraphiteMockServer graphiteMockServer; @@ -25,6 +26,7 @@ public class GraphitePluginIntegrationTest { private String clusterName = UUID.randomUUID().toString().replaceAll("-", ""); private String index = UUID.randomUUID().toString().replaceAll("-", ""); private String type = UUID.randomUUID().toString().replaceAll("-", ""); + private String defaultPrefix = "elasticsearch." + clusterName; private Node node; @Before @@ -43,22 +45,23 @@ public void stopGraphiteServer() throws Exception { @Test public void testThatIndexingResultsInMonitoring() throws Exception { - node = createNode(clusterName, GRAPHITE_SERVER_PORT, "1s"); + node = createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null); IndexResponse indexResponse = indexElement(node, index, type, "value"); assertThat(indexResponse.getId(), is(notNullValue())); Thread.sleep(2000); ensureValidKeyNames(); - assertGraphiteMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing._all.indexCount 1"); - assertGraphiteMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing." + type + ".indexCount 1"); - assertGraphiteMetricIsContained("elasticsearch." + clusterName + ".node.jvm.threads.peakCount "); + assertGraphiteMetricIsContained(defaultPrefix + ".indexes." + index + ".id.0.indexing._all.indexCount 1"); + assertGraphiteMetricIsContained(defaultPrefix + ".indexes." + index + ".id.0.indexing." + type + ".indexCount 1"); + assertGraphiteMetricIsContained(defaultPrefix + ".node.jvm.threads.peakCount "); } + @Test public void testThatFieldExclusionWorks() throws Exception { String excludeRegex = ".*\\.peakCount"; - node = createNode(clusterName, GRAPHITE_SERVER_PORT, "1s", null, excludeRegex); + node = createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null, null, excludeRegex); IndexResponse indexResponse = indexElement(node, index, type, "value"); assertThat(indexResponse.getId(), is(notNullValue())); @@ -66,14 +69,14 @@ public void testThatFieldExclusionWorks() throws Exception { Thread.sleep(2000); ensureValidKeyNames(); - assertGraphiteMetricIsNotContained("elasticsearch." + clusterName + ".node.jvm.threads.peakCount "); + assertGraphiteMetricIsNotContained(defaultPrefix + ".node.jvm.threads.peakCount "); } @Test public void testThatFieldInclusionWinsOverExclusion() throws Exception { String excludeRegex = ".*" + clusterName + ".*"; String includeRegex = ".*\\.peakCount"; - node = createNode(clusterName, GRAPHITE_SERVER_PORT, "1s", includeRegex, excludeRegex); + node = createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null, includeRegex, excludeRegex); IndexResponse indexResponse = indexElement(node, index, type, "value"); assertThat(indexResponse.getId(), is(notNullValue())); @@ -81,26 +84,28 @@ public void testThatFieldInclusionWinsOverExclusion() throws Exception { Thread.sleep(2000); ensureValidKeyNames(); - assertGraphiteMetricIsNotContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing._all.indexCount 1"); - assertGraphiteMetricIsContained("elasticsearch." + clusterName + ".node.jvm.threads.peakCount "); + assertGraphiteMetricIsNotContained(defaultPrefix + ".indexes." + index + ".id.0.indexing._all.indexCount 1"); + assertGraphiteMetricIsContained(defaultPrefix + ".node.jvm.threads.peakCount "); } @Test(expected = ProvisionException.class) public void testThatBrokenRegexLeadsToException() throws Exception { String excludeRegex = "*.peakCount"; - createNode(clusterName, GRAPHITE_SERVER_PORT, "1s", null, excludeRegex); + createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null, null, excludeRegex); } - @Test public void masterFailOverShouldWork() throws Exception { - node = createNode(clusterName, GRAPHITE_SERVER_PORT, "1s"); + // Generate a new cluster not to interfer with the precedents tests String clusterName = UUID.randomUUID().toString().replaceAll("-", ""); + String defaultPrefix = "elasticsearch." + clusterName; + + node = createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null); IndexResponse indexResponse = indexElement(node, index, type, "value"); assertThat(indexResponse.getId(), is(notNullValue())); Node origNode = node; - node = createNode(clusterName, GRAPHITE_SERVER_PORT, "1s"); + node = createNode(clusterName, true, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", null); graphiteMockServer.content.clear(); origNode.stop(); indexResponse = indexElement(node, index, type, "value"); @@ -108,7 +113,48 @@ public void masterFailOverShouldWork() throws Exception { // wait for master fail over and writing to graph reporter Thread.sleep(2000); - assertGraphiteMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing._all.indexCount 1"); + assertGraphiteMetricIsContained(defaultPrefix + ".indexes." + index + ".id.0.indexing._all.indexCount 1"); + } + + @Test + public void testThatNotMasterNodeIsMonitoredAsWell() throws Exception { + // Generate a new cluster not to interfer with the precedents tests + String clusterName = UUID.randomUUID().toString().replaceAll("-", ""); + String defaultPrefix = "elasticsearch." + clusterName; + + // create a master node without the plugin disabled + Node masterNode = createNode(clusterName, true); + graphiteMockServer.content.clear(); + Thread.sleep(2000); + // create a non-master node with the plugin enabled, + // and a specific prefix to be sure that the metrics are emitted by that node + String specificPrefix = defaultPrefix + ".dataNode"; + node = createNode(clusterName, false, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", specificPrefix); + Thread.sleep(2000); + + assertGraphiteMetricIsContained(specificPrefix + ".node.jvm.uptime "); + } + + @Test + public void testThatNotMasterNodeIsNotMonitoredWhenNoMasterIsFound() throws Exception { + // Generate a new cluster not to interfer with the precedents tests + String clusterName = UUID.randomUUID().toString().replaceAll("-", ""); + String defaultPrefix = "elasticsearch." + clusterName; + + // create a non-master node with the plugin enabled, + // and no metric should be reported until a master is elected + String specificPrefix = defaultPrefix + ".dataNode"; + graphiteMockServer.content.clear(); + node = createNode(clusterName, false, GRAPHITE_SERVER_HOST, GRAPHITE_SERVER_PORT, "1s", specificPrefix); + Thread.sleep(4000); + assertGraphiteMetricIsNotContained(specificPrefix + ".node.jvm.uptime "); + + // create a master node to be joigned... + graphiteMockServer.content.clear(); + Node masterNode = createNode(clusterName, true); + Thread.sleep(4000); + //the dataNode should now start reporting metrics + assertGraphiteMetricIsContained(specificPrefix + ".node.jvm.uptime "); } // the stupid hamcrest matchers have compile erros depending whether they run on java6 or java7, so I rolled my own version @@ -135,4 +181,4 @@ private IndexResponse indexElement(Node node, String index, String type, String setSource("field", fieldValue) .execute().actionGet(); } -} +} \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/module/graphite/test/NodeTestHelper.java b/src/test/java/org/elasticsearch/module/graphite/test/NodeTestHelper.java index 3969429..05fd0be 100644 --- a/src/test/java/org/elasticsearch/module/graphite/test/NodeTestHelper.java +++ b/src/test/java/org/elasticsearch/module/graphite/test/NodeTestHelper.java @@ -10,22 +10,34 @@ public class NodeTestHelper { - public static Node createNode(String clusterName, int graphitePort, String refreshInterval) throws IOException { - return createNode(clusterName, graphitePort, refreshInterval, null, null); + public static Node createNode(String clusterName, boolean nodeMaster) throws IOException { + return createNode(clusterName, nodeMaster, null, 0, null, null, null, null); } - public static Node createNode(String clusterName, int graphitePort, String refreshInterval, String includeRegex, + public static Node createNode(String clusterName, boolean nodeMaster, String graphiteHost, int graphitePort, String refreshInterval, String graphitePrefix) throws IOException { + return createNode(clusterName, nodeMaster, graphiteHost, graphitePort, refreshInterval, graphitePrefix, null, null); + } + + public static Node createNode(String clusterName, boolean nodeMaster, String graphiteHost, int graphitePort, String refreshInterval, String graphitePrefix, String includeRegex, String excludeRegex) throws IOException { ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder(); settingsBuilder.put("gateway.type", "none"); settingsBuilder.put("cluster.name", clusterName); + settingsBuilder.put("node.master", nodeMaster); settingsBuilder.put("index.number_of_shards", 1); settingsBuilder.put("index.number_of_replicas", 1); - - settingsBuilder.put("metrics.graphite.host", "localhost"); - settingsBuilder.put("metrics.graphite.port", graphitePort); - settingsBuilder.put("metrics.graphite.every", refreshInterval); + settingsBuilder.put("discovery.zen.ping.multicast.enabled", false); + settingsBuilder.put("discovery.zen.ping.unicast.hosts", "localhost:9300,localhost:9301,localhost:9302"); + + if (Strings.hasLength(graphiteHost)) { + settingsBuilder.put("metrics.graphite.host", graphiteHost); + settingsBuilder.put("metrics.graphite.port", graphitePort); + settingsBuilder.put("metrics.graphite.every", refreshInterval); + if (Strings.hasLength(graphitePrefix)) { + settingsBuilder.put("metrics.graphite.prefix", graphitePrefix); + } + } if (Strings.hasLength(includeRegex)) { settingsBuilder.put("metrics.graphite.include", includeRegex); @@ -39,4 +51,4 @@ public static Node createNode(String clusterName, int graphitePort, String refre return NodeBuilder.nodeBuilder().settings(settingsBuilder.build()).node(); } -} +} \ No newline at end of file diff --git a/src/test/resources/logging.yml b/src/test/resources/logging.yml index 4c9bd6b..181f178 100644 --- a/src/test/resources/logging.yml +++ b/src/test/resources/logging.yml @@ -1,7 +1,7 @@ rootLogger: INFO, console logger: - service.graphite: DEBUG + org.elasticsearch.service.graphite: DEBUG plugin.graphite.GraphitePlugin: DEBUG appender: @@ -9,4 +9,4 @@ appender: type: console layout: type: consolePattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" \ No newline at end of file