Support Datadog distribution metric type (#132)

The Datadog client now has the ability to send histogram metrics as the
Datadog-specific distribution metric type. The Datadog client
configuration has a new `DistributionPrefixes` item which specifies the
prefixes of metric names that, when reported as histograms to the stats
library, are to be sent as distributions instead. For example, when the
prefix list is set to `{ "dist_" }`, then any histogram metric whose
name begins with "dist_" is sent as a distribution; all other histograms
are sent as ordinary histograms, as before.

The default configuration sends no histograms as distributions.

Author: bhavanki

datadog/client.go

@@ -25,13 +25,17 @@ const (
 	MaxBufferSize = 65507
 )
 
-// DefaultFilter is the default tag to filter before sending to
-// datadog. Using the request path as a tag can overwhelm datadog's
-// servers if there are too many unique routes due to unique IDs being a
-// part of the path. Only change the default filter if there is a static
-// number of routes.
 var (
+	// DefaultFilters are the default tags to filter before sending to
+	// datadog. Using the request path as a tag can overwhelm datadog's
+	// servers if there are too many unique routes due to unique IDs being a
+	// part of the path. Only change the default filters if there are a static
+	// number of routes.
 	DefaultFilters = []string{"http_req_path"}
+
+	// DefaultDistributionPrefixes is the default set of name prefixes for
+	// metrics to be sent as distributions instead of as histograms.
+	DefaultDistributionPrefixes = []string{}
 )
 
 // The ClientConfig type is used to configure datadog clients.
@@ -44,6 +48,10 @@ type ClientConfig struct {
 
 	// List of tags to filter. If left nil is set to DefaultFilters.
 	Filters []string
+
+	// Set of name prefixes for metrics to be sent as distributions instead of
+	// as histograms.
+	DistributionPrefixes []string
 }
 
 // Client represents an datadog client that implements the stats.Handler
@@ -77,6 +85,10 @@ func NewClientWith(config ClientConfig) *Client {
 		config.Filters = DefaultFilters
 	}
 
+	if config.DistributionPrefixes == nil {
+		config.DistributionPrefixes = DefaultDistributionPrefixes
+	}
+
 	// transform filters from array to map
 	filterMap := make(map[string]struct{})
 	for _, f := range config.Filters {
@@ -85,7 +97,8 @@ func NewClientWith(config ClientConfig) *Client {
 
 	c := &Client{
 		serializer: serializer{
-			filters: filterMap,
+			filters:      filterMap,
+			distPrefixes: config.DistributionPrefixes,
 		},
 	}
 
@@ -124,14 +137,15 @@ func (c *Client) Close() error {
 }
 
 type serializer struct {
-	conn       net.Conn
-	bufferSize int
-	filters    map[string]struct{}
+	conn         net.Conn
+	bufferSize   int
+	filters      map[string]struct{}
+	distPrefixes []string
 }
 
 func (s *serializer) AppendMeasures(b []byte, _ time.Time, measures ...stats.Measure) []byte {
 	for _, m := range measures {
-		b = AppendMeasureFiltered(b, m, s.filters)
+		b = AppendMeasureFiltered(b, m, s.filters, s.distPrefixes)
 	}
 	return b
 }

datadog/client_test.go

@@ -35,6 +35,29 @@ func TestClient(t *testing.T) {
 	}
 }
 
+func TestClientWithDistributionPrefixes(t *testing.T) {
+	client := NewClientWith(ClientConfig{
+		Address:              DefaultAddress,
+		DistributionPrefixes: []string{"dist_"},
+	})
+
+	client.HandleMeasures(time.Time{}, stats.Measure{
+		Name: "request",
+		Fields: []stats.Field{
+			{Name: "count", Value: stats.ValueOf(5)},
+			stats.MakeField("dist_rtt", stats.ValueOf(100*time.Millisecond), stats.Histogram),
+		},
+		Tags: []stats.Tag{
+			stats.T("answer", "42"),
+			stats.T("hello", "world"),
+		},
+	})
+
+	if err := client.Close(); err != nil {
+		t.Error(err)
+	}
+}
+
 func TestClientWriteLargeMetrics(t *testing.T) {
 	const data = `main.http.error.count:0|c|#http_req_content_charset:,http_req_content_endoing:,http_req_content_type:,http_req_host:localhost:3011,http_req_method:GET,http_req_protocol:HTTP/1.1,http_req_transfer_encoding:identity
 main.http.message.count:1|c|#http_req_content_charset:,http_req_content_endoing:,http_req_content_type:,http_req_host:localhost:3011,http_req_method:GET,http_req_protocol:HTTP/1.1,http_req_transfer_encoding:identity,operation:read,type:request

datadog/measure.go

@@ -3,20 +3,24 @@ package datadog
 import (
 	"math"
 	"strconv"
+	"strings"
 
 	"github.com/segmentio/stats/v4"
 )
 
+// Datagram format: https://docs.datadoghq.com/developers/dogstatsd/datagram_shell
+
 // AppendMeasure is a formatting routine to append the dogstatsd protocol
 // representation of a measure to a memory buffer.
 func AppendMeasure(b []byte, m stats.Measure) []byte {
-	return AppendMeasureFiltered(b, m, nil)
+	return AppendMeasureFiltered(b, m, nil, []string{})
 }
 
 // AppendMeasureFiltered is a formatting routine to append the dogstatsd protocol
 // representation of a measure to a memory buffer. Tags listed in the filters map
 // are removed. (some tags may not be suitable for submission to DataDog)
-func AppendMeasureFiltered(b []byte, m stats.Measure, filters map[string]struct{}) []byte {
+func AppendMeasureFiltered(b []byte, m stats.Measure, filters map[string]struct{},
+	distPrefixes []string) []byte {
 	for _, field := range m.Fields {
 		b = append(b, m.Name...)
 		if len(field.Name) != 0 {
@@ -50,7 +54,11 @@ func AppendMeasureFiltered(b []byte, m stats.Measure, filters map[string]struct{
 		case stats.Gauge:
 			b = append(b, '|', 'g')
 		default:
-			b = append(b, '|', 'h')
+			if sendDist(field.Name, distPrefixes) {
+				b = append(b, '|', 'd')
+			} else {
+				b = append(b, '|', 'h')
+			}
 		}
 
 		if n := len(m.Tags); n != 0 {
@@ -86,3 +94,15 @@ func normalizeFloat(f float64) float64 {
 		return f
 	}
 }
+
+func sendDist(name string, distPrefixes []string) bool {
+	if distPrefixes == nil {
+		return false
+	}
+	for _, prefix := range distPrefixes {
+		if strings.HasPrefix(name, prefix) {
+			return true
+		}
+	}
+	return false
+}

datadog/measure_test.go

@@ -9,8 +9,9 @@ import (
 
 var (
 	testMeasures = []struct {
-		m stats.Measure
-		s string
+		m  stats.Measure
+		s  string
+		dp []string
 	}{
 		{
 			m: stats.Measure{
@@ -21,6 +22,7 @@ var (
 			},
 			s: `request.count:5|c
 `,
+			dp: []string{},
 		},
 
 		{
@@ -38,18 +40,73 @@ var (
 			s: `request.count:5|c|#answer:42,hello:world
 request.rtt:0.1|h|#answer:42,hello:world
 `,
+			dp: []string{},
+		},
+
+		{
+			m: stats.Measure{
+				Name: "request",
+				Fields: []stats.Field{
+					stats.MakeField("dist_rtt", 100*time.Millisecond, stats.Histogram),
+				},
+				Tags: []stats.Tag{
+					stats.T("answer", "42"),
+					stats.T("hello", "world"),
+				},
+			},
+			s: `request.dist_rtt:0.1|d|#answer:42,hello:world
+`,
+			dp: []string{"dist_"},
 		},
 	}
 )
 
 func TestAppendMeasure(t *testing.T) {
 	for _, test := range testMeasures {
 		t.Run(test.s, func(t *testing.T) {
-			if s := string(AppendMeasure(nil, test.m)); s != test.s {
+			if s := string(AppendMeasureFiltered(nil, test.m, nil, test.dp)); s != test.s {
 				t.Error("bad metric representation:")
 				t.Log("expected:", test.s)
 				t.Log("found:   ", s)
 			}
 		})
 	}
 }
+
+var (
+	testDistNames = []struct {
+		n string
+		d bool
+	}{
+		{
+			n: "name",
+			d: false,
+		},
+		{
+			n: "",
+			d: false,
+		},
+		{
+			n: "dist_name",
+			d: true,
+		},
+		{
+			n: "distname",
+			d: false,
+		},
+	}
+	distPrefixes = []string{"dist_"}
+)
+
+func TestSendDist(t *testing.T) {
+	for _, test := range testDistNames {
+		t.Run(test.n, func(t *testing.T) {
+			a := sendDist(test.n, distPrefixes)
+			if a != test.d {
+				t.Error("distribution name detection incorrect:")
+				t.Log("expected:", test.d)
+				t.Log("found:   ", a)
+			}
+		})
+	}
+}

datadog/metric.go

@@ -8,15 +8,16 @@ import (
 )
 
 // MetricType is an enumeration providing symbols to represent the different
-// metric types upported by datadog.
+// metric types supported by datadog.
 type MetricType string
 
 // Metric Types.
 const (
-	Counter   MetricType = "c"
-	Gauge     MetricType = "g"
-	Histogram MetricType = "h"
-	Unknown   MetricType = "?"
+	Counter      MetricType = "c"
+	Gauge        MetricType = "g"
+	Histogram    MetricType = "h"
+	Distribution MetricType = "d"
+	Unknown      MetricType = "?"
 )
 
 // The Metric type is a representation of the metrics supported by datadog.

datadog/metric_test.go

@@ -101,6 +101,28 @@ var testMetrics = []struct {
 		},
 	},
 
+	{
+		s: "song.length:240|d|@0.5\n",
+		m: Metric{
+			Type:  Distribution,
+			Name:  "song.length",
+			Value: 240,
+			Rate:  0.5,
+			Tags:  nil,
+		},
+	},
+
+	{
+		s: "users.uniques:1234|d\n",
+		m: Metric{
+			Type:  Distribution,
+			Name:  "users.uniques",
+			Value: 1234,
+			Rate:  1,
+			Tags:  nil,
+		},
+	},
+
 	{
 		s: "users.online:1|c|#country:china\n",
 		m: Metric{