commit e5fa7cef3ec50a868470e0e601120b7269cdad45 Author: brettlangdon Date: Sun Jan 24 18:51:06 2016 -0500 Squashed 'vendor/github.com/DataDog/datadog-go/' content from commit bc97e07 git-subtree-dir: vendor/github.com/DataDog/datadog-go git-subtree-split: bc97e0770ad4edae1c9dc14beb40b79b2dde32f8 diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a162f86 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: go + +go: + - 1.4 + - 1.5 + +script: + - go test -v ./... diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..97cd06d --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2015 Datadog, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..dce6271 --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# Overview + +Packages in `datadog-go` provide Go clients for various APIs at [DataDog](http://datadoghq.com). + +## Statsd + +[![Godoc](http://img.shields.io/badge/godoc-reference-blue.svg?style=flat)](https://godoc.org/github.com/DataDog/datadog-go/statsd) +[![license](http://img.shields.io/badge/license-MIT-red.svg?style=flat)](http://opensource.org/licenses/MIT) + +The [statsd](https://github.com/DataDog/datadog-go/tree/master/statsd) package provides a client for +[dogstatsd](http://docs.datadoghq.com/guides/dogstatsd/): + +```go +import "github.com/DataDog/datadog-go/statsd" + +func main() { + c, err := statsd.New("127.0.0.1:8125") + if err != nil { + log.Fatal(err) + } + // prefix every metric with the app name + c.Namespace = "flubber." + // send the EC2 availability zone as a tag with every metric + c.Tags = append(c.Tags, "us-east-1a") + err = c.Gauge("request.duration", 1.2, nil, 1) + // ... +} +``` + +## License + +All code distributed under the [MIT License](http://opensource.org/licenses/MIT) unless otherwise specified. diff --git a/statsd/README.md b/statsd/README.md new file mode 100644 index 0000000..c3b462f --- /dev/null +++ b/statsd/README.md @@ -0,0 +1,45 @@ +## Overview + +Package `statsd` provides a Go [dogstatsd](http://docs.datadoghq.com/guides/dogstatsd/) client. Dogstatsd extends Statsd, adding tags +and histograms. + +## Get the code + + $ go get github.com/DataDog/datadog-go/statsd + +## Usage + +```go +// Create the client +c, err := statsd.New("127.0.0.1:8125") +if err != nil { + log.Fatal(err) +} +// Prefix every metric with the app name +c.Namespace = "flubber." +// Send the EC2 availability zone as a tag with every metric +c.Tags = append(c.Tags, "us-east-1a") +err = c.Gauge("request.duration", 1.2, nil, 1) +``` + +## Buffering Client + +Dogstatsd accepts packets with multiple statsd payloads in them. Using the BufferingClient via `NewBufferingClient` will buffer up commands and send them when the buffer is reached or after 100msec. + +## Development + +Run the tests with: + + $ go test + +## Documentation + +Please see: http://godoc.org/github.com/DataDog/datadog-go/statsd + +## License + +go-dogstatsd is released under the [MIT license](http://www.opensource.org/licenses/mit-license.php). + +## Credits + +Original code by [ooyala](https://github.com/ooyala/go-dogstatsd). diff --git a/statsd/statsd.go b/statsd/statsd.go new file mode 100644 index 0000000..4038b89 --- /dev/null +++ b/statsd/statsd.go @@ -0,0 +1,353 @@ +// Copyright 2013 Ooyala, Inc. + +/* +Package statsd provides a Go dogstatsd client. Dogstatsd extends the popular statsd, +adding tags and histograms and pushing upstream to Datadog. + +Refer to http://docs.datadoghq.com/guides/dogstatsd/ for information about DogStatsD. + +Example Usage: + + // Create the client + c, err := statsd.New("127.0.0.1:8125") + if err != nil { + log.Fatal(err) + } + // Prefix every metric with the app name + c.Namespace = "flubber." + // Send the EC2 availability zone as a tag with every metric + c.Tags = append(c.Tags, "us-east-1a") + err = c.Gauge("request.duration", 1.2, nil, 1) + +statsd is based on go-statsd-client. +*/ +package statsd + +import ( + "bytes" + "fmt" + "math/rand" + "net" + "strconv" + "strings" + "sync" + "time" +) + +// A Client is a handle for sending udp messages to dogstatsd. It is safe to +// use one Client from multiple goroutines simultaneously. +type Client struct { + conn net.Conn + // Namespace to prepend to all statsd calls + Namespace string + // Tags are global tags to be added to every statsd call + Tags []string + // BufferLength is the length of the buffer in commands. + bufferLength int + flushTime time.Duration + commands []string + stop bool + sync.Mutex +} + +// New returns a pointer to a new Client given an addr in the format "hostname:port". +func New(addr string) (*Client, error) { + udpAddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + return nil, err + } + conn, err := net.DialUDP("udp", nil, udpAddr) + if err != nil { + return nil, err + } + client := &Client{conn: conn} + return client, nil +} + +// NewBuffered returns a Client that buffers its output and sends it in chunks. +// Buflen is the length of the buffer in number of commands. +func NewBuffered(addr string, buflen int) (*Client, error) { + client, err := New(addr) + if err != nil { + return nil, err + } + client.bufferLength = buflen + client.commands = make([]string, 0, buflen) + client.flushTime = time.Millisecond * 100 + go client.watch() + return client, nil +} + +// format a message from its name, value, tags and rate. Also adds global +// namespace and tags. +func (c *Client) format(name, value string, tags []string, rate float64) string { + var buf bytes.Buffer + if c.Namespace != "" { + buf.WriteString(c.Namespace) + } + buf.WriteString(name) + buf.WriteString(":") + buf.WriteString(value) + if rate < 1 { + buf.WriteString(`|@`) + buf.WriteString(strconv.FormatFloat(rate, 'f', -1, 64)) + } + + tags = append(c.Tags, tags...) + if len(tags) > 0 { + buf.WriteString("|#") + buf.WriteString(tags[0]) + for _, tag := range tags[1:] { + buf.WriteString(",") + buf.WriteString(tag) + } + } + return buf.String() +} + +func (c *Client) watch() { + for _ = range time.Tick(c.flushTime) { + if c.stop { + return + } + c.Lock() + if len(c.commands) > 0 { + // FIXME: eating error here + c.flush() + } + c.Unlock() + } +} + +func (c *Client) append(cmd string) error { + c.Lock() + c.commands = append(c.commands, cmd) + // if we should flush, lets do it + if len(c.commands) == c.bufferLength { + if err := c.flush(); err != nil { + c.Unlock() + return err + } + } + c.Unlock() + return nil +} + +// flush the commands in the buffer. Lock must be held by caller. +func (c *Client) flush() error { + data := strings.Join(c.commands, "\n") + _, err := c.conn.Write([]byte(data)) + // clear the slice with a slice op, doesn't realloc + c.commands = c.commands[:0] + return err +} + +func (c *Client) sendMsg(msg string) error { + // if this client is buffered, then we'll just append this + if c.bufferLength > 0 { + return c.append(msg) + } + c.Lock() + _, err := c.conn.Write([]byte(msg)) + c.Unlock() + return err +} + +// send handles sampling and sends the message over UDP. It also adds global namespace prefixes and tags. +func (c *Client) send(name, value string, tags []string, rate float64) error { + if c == nil { + return nil + } + if rate < 1 && rand.Float64() > rate { + return nil + } + data := c.format(name, value, tags, rate) + return c.sendMsg(data) +} + +// Gauge measures the value of a metric at a particular time. +func (c *Client) Gauge(name string, value float64, tags []string, rate float64) error { + stat := fmt.Sprintf("%f|g", value) + return c.send(name, stat, tags, rate) +} + +// Count tracks how many times something happened per second. +func (c *Client) Count(name string, value int64, tags []string, rate float64) error { + stat := fmt.Sprintf("%d|c", value) + return c.send(name, stat, tags, rate) +} + +// Histogram tracks the statistical distribution of a set of values. +func (c *Client) Histogram(name string, value float64, tags []string, rate float64) error { + stat := fmt.Sprintf("%f|h", value) + return c.send(name, stat, tags, rate) +} + +// Set counts the number of unique elements in a group. +func (c *Client) Set(name string, value string, tags []string, rate float64) error { + stat := fmt.Sprintf("%s|s", value) + return c.send(name, stat, tags, rate) +} + +// TimeInMilliseconds sends timing information in milliseconds. +// It is flushed by statsd with percentiles, mean and other info (https://github.com/etsy/statsd/blob/master/docs/metric_types.md#timing) +func (c *Client) TimeInMilliseconds(name string, value float64, tags []string, rate float64) error { + stat := fmt.Sprintf("%f|ms", value) + return c.send(name, stat, tags, rate) +} + +// Event sends the provided Event. +func (c *Client) Event(e *Event) error { + stat, err := e.Encode(c.Tags...) + if err != nil { + return err + } + return c.sendMsg(stat) +} + +// SimpleEvent sends an event with the provided title and text. +func (c *Client) SimpleEvent(title, text string) error { + e := NewEvent(title, text) + return c.Event(e) +} + +// Close the client connection. +func (c *Client) Close() error { + if c == nil { + return nil + } + c.stop = true + return c.conn.Close() +} + +// Events support + +type eventAlertType string + +const ( + // Info is the "info" AlertType for events + Info eventAlertType = "info" + // Error is the "error" AlertType for events + Error eventAlertType = "error" + // Warning is the "warning" AlertType for events + Warning eventAlertType = "warning" + // Success is the "success" AlertType for events + Success eventAlertType = "success" +) + +type eventPriority string + +const ( + // Normal is the "normal" Priority for events + Normal eventPriority = "normal" + // Low is the "low" Priority for events + Low eventPriority = "low" +) + +// An Event is an object that can be posted to your DataDog event stream. +type Event struct { + // Title of the event. Required. + Title string + // Text is the description of the event. Required. + Text string + // Timestamp is a timestamp for the event. If not provided, the dogstatsd + // server will set this to the current time. + Timestamp time.Time + // Hostname for the event. + Hostname string + // AggregationKey groups this event with others of the same key. + AggregationKey string + // Priority of the event. Can be statsd.Low or statsd.Normal. + Priority eventPriority + // SourceTypeName is a source type for the event. + SourceTypeName string + // AlertType can be statsd.Info, statsd.Error, statsd.Warning, or statsd.Success. + // If absent, the default value applied by the dogstatsd server is Info. + AlertType eventAlertType + // Tags for the event. + Tags []string +} + +// NewEvent creates a new event with the given title and text. Error checking +// against these values is done at send-time, or upon running e.Check. +func NewEvent(title, text string) *Event { + return &Event{ + Title: title, + Text: text, + } +} + +// Check verifies that an event is valid. +func (e Event) Check() error { + if len(e.Title) == 0 { + return fmt.Errorf("statsd.Event title is required") + } + if len(e.Text) == 0 { + return fmt.Errorf("statsd.Event text is required") + } + return nil +} + +// Encode returns the dogstatsd wire protocol representation for an event. +// Tags may be passed which will be added to the encoded output but not to +// the Event's list of tags, eg. for default tags. +func (e Event) Encode(tags ...string) (string, error) { + err := e.Check() + if err != nil { + return "", err + } + var buffer bytes.Buffer + buffer.WriteString("_e{") + buffer.WriteString(strconv.FormatInt(int64(len(e.Title)), 10)) + buffer.WriteRune(',') + buffer.WriteString(strconv.FormatInt(int64(len(e.Text)), 10)) + buffer.WriteString("}:") + buffer.WriteString(e.Title) + buffer.WriteRune('|') + buffer.WriteString(e.Text) + + if !e.Timestamp.IsZero() { + buffer.WriteString("|d:") + buffer.WriteString(strconv.FormatInt(int64(e.Timestamp.Unix()), 10)) + } + + if len(e.Hostname) != 0 { + buffer.WriteString("|h:") + buffer.WriteString(e.Hostname) + } + + if len(e.AggregationKey) != 0 { + buffer.WriteString("|k:") + buffer.WriteString(e.AggregationKey) + + } + + if len(e.Priority) != 0 { + buffer.WriteString("|p:") + buffer.WriteString(string(e.Priority)) + } + + if len(e.SourceTypeName) != 0 { + buffer.WriteString("|s:") + buffer.WriteString(e.SourceTypeName) + } + + if len(e.AlertType) != 0 { + buffer.WriteString("|t:") + buffer.WriteString(string(e.AlertType)) + } + + if len(tags)+len(e.Tags) > 0 { + all := make([]string, 0, len(tags)+len(e.Tags)) + all = append(all, tags...) + all = append(all, e.Tags...) + buffer.WriteString("|#") + buffer.WriteString(all[0]) + for _, tag := range all[1:] { + buffer.WriteString(",") + buffer.WriteString(tag) + } + } + + return buffer.String(), nil +} diff --git a/statsd/statsd_test.go b/statsd/statsd_test.go new file mode 100644 index 0000000..1f12fc0 --- /dev/null +++ b/statsd/statsd_test.go @@ -0,0 +1,312 @@ +// Copyright 2013 Ooyala, Inc. + +package statsd + +import ( + "fmt" + "io" + "net" + "reflect" + "strings" + "testing" +) + +var dogstatsdTests = []struct { + GlobalNamespace string + GlobalTags []string + Method string + Metric string + Value interface{} + Tags []string + Rate float64 + Expected string +}{ + {"", nil, "Gauge", "test.gauge", 1.0, nil, 1.0, "test.gauge:1.000000|g"}, + {"", nil, "Gauge", "test.gauge", 1.0, nil, 0.999999, "test.gauge:1.000000|g|@0.999999"}, + {"", nil, "Gauge", "test.gauge", 1.0, []string{"tagA"}, 1.0, "test.gauge:1.000000|g|#tagA"}, + {"", nil, "Gauge", "test.gauge", 1.0, []string{"tagA", "tagB"}, 1.0, "test.gauge:1.000000|g|#tagA,tagB"}, + {"", nil, "Gauge", "test.gauge", 1.0, []string{"tagA"}, 0.999999, "test.gauge:1.000000|g|@0.999999|#tagA"}, + {"", nil, "Count", "test.count", int64(1), []string{"tagA"}, 1.0, "test.count:1|c|#tagA"}, + {"", nil, "Count", "test.count", int64(-1), []string{"tagA"}, 1.0, "test.count:-1|c|#tagA"}, + {"", nil, "Histogram", "test.histogram", 2.3, []string{"tagA"}, 1.0, "test.histogram:2.300000|h|#tagA"}, + {"", nil, "Set", "test.set", "uuid", []string{"tagA"}, 1.0, "test.set:uuid|s|#tagA"}, + {"flubber.", nil, "Set", "test.set", "uuid", []string{"tagA"}, 1.0, "flubber.test.set:uuid|s|#tagA"}, + {"", []string{"tagC"}, "Set", "test.set", "uuid", []string{"tagA"}, 1.0, "test.set:uuid|s|#tagC,tagA"}, +} + +func assertNotPanics(t *testing.T, f func()) { + defer func() { + if r := recover(); r != nil { + t.Fatal(r) + } + }() + f() +} + +func TestClient(t *testing.T) { + addr := "localhost:1201" + udpAddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + t.Fatal(err) + } + + server, err := net.ListenUDP("udp", udpAddr) + if err != nil { + t.Fatal(err) + } + defer server.Close() + + client, err := New(addr) + if err != nil { + t.Fatal(err) + } + + for _, tt := range dogstatsdTests { + client.Namespace = tt.GlobalNamespace + client.Tags = tt.GlobalTags + method := reflect.ValueOf(client).MethodByName(tt.Method) + e := method.Call([]reflect.Value{ + reflect.ValueOf(tt.Metric), + reflect.ValueOf(tt.Value), + reflect.ValueOf(tt.Tags), + reflect.ValueOf(tt.Rate)})[0] + errInter := e.Interface() + if errInter != nil { + t.Fatal(errInter.(error)) + } + + bytes := make([]byte, 1024) + n, err := server.Read(bytes) + if err != nil { + t.Fatal(err) + } + message := bytes[:n] + if string(message) != tt.Expected { + t.Errorf("Expected: %s. Actual: %s", tt.Expected, string(message)) + } + } +} + +func TestBufferedClient(t *testing.T) { + addr := "localhost:1201" + udpAddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + t.Fatal(err) + } + + server, err := net.ListenUDP("udp", udpAddr) + if err != nil { + t.Fatal(err) + } + defer server.Close() + + conn, err := net.DialUDP("udp", nil, udpAddr) + if err != nil { + t.Fatal(err) + } + + bufferLength := 5 + client := &Client{ + conn: conn, + commands: make([]string, 0, bufferLength), + bufferLength: bufferLength, + } + + client.Namespace = "foo." + client.Tags = []string{"dd:2"} + + client.Count("cc", 1, nil, 1) + client.Gauge("gg", 10, nil, 1) + client.Histogram("hh", 1, nil, 1) + client.Set("ss", "ss", nil, 1) + + if len(client.commands) != 4 { + t.Errorf("Expected client to have buffered 4 commands, but found %d\n", len(client.commands)) + } + + client.Set("ss", "xx", nil, 1) + err = client.flush() + if err != nil { + t.Errorf("Error sending: %s", err) + } + + if len(client.commands) != 0 { + t.Errorf("Expecting send to flush commands, but found %d\n", len(client.commands)) + } + + buffer := make([]byte, 4096) + n, err := io.ReadAtLeast(server, buffer, 1) + result := string(buffer[:n]) + + if err != nil { + t.Error(err) + } + + expected := []string{ + `foo.cc:1|c|#dd:2`, + `foo.gg:10.000000|g|#dd:2`, + `foo.hh:1.000000|h|#dd:2`, + `foo.ss:ss|s|#dd:2`, + `foo.ss:xx|s|#dd:2`, + } + + for i, res := range strings.Split(result, "\n") { + if res != expected[i] { + t.Errorf("Got `%s`, expected `%s`", res, expected[i]) + } + } + + client.Event(&Event{Title: "title1", Text: "text1", Priority: Normal, AlertType: Success, Tags: []string{"tagg"}}) + client.SimpleEvent("event1", "text1") + + if len(client.commands) != 2 { + t.Errorf("Expected to find %d commands, but found %d\n", 2, len(client.commands)) + } + + err = client.flush() + + if err != nil { + t.Errorf("Error sending: %s", err) + } + + if len(client.commands) != 0 { + t.Errorf("Expecting send to flush commands, but found %d\n", len(client.commands)) + } + + buffer = make([]byte, 1024) + n, err = io.ReadAtLeast(server, buffer, 1) + result = string(buffer[:n]) + + if err != nil { + t.Error(err) + } + + if n == 0 { + t.Errorf("Read 0 bytes but expected more.") + } + + expected = []string{ + `_e{6,5}:title1|text1|p:normal|t:success|#dd:2,tagg`, + `_e{6,5}:event1|text1|#dd:2`, + } + + for i, res := range strings.Split(result, "\n") { + if res != expected[i] { + t.Errorf("Got `%s`, expected `%s`", res, expected[i]) + } + } + +} + +func TestNilSafe(t *testing.T) { + var c *Client + assertNotPanics(t, func() { c.Close() }) + assertNotPanics(t, func() { c.Count("", 0, nil, 1) }) + assertNotPanics(t, func() { c.Histogram("", 0, nil, 1) }) + assertNotPanics(t, func() { c.Gauge("", 0, nil, 1) }) + assertNotPanics(t, func() { c.Set("", "", nil, 1) }) + assertNotPanics(t, func() { c.send("", "", nil, 1) }) +} + +func TestEvents(t *testing.T) { + matrix := []struct { + event *Event + encoded string + }{ + { + NewEvent("Hello", "Something happened to my event"), + `_e{5,30}:Hello|Something happened to my event`, + }, { + &Event{Title: "hi", Text: "okay", AggregationKey: "foo"}, + `_e{2,4}:hi|okay|k:foo`, + }, { + &Event{Title: "hi", Text: "okay", AggregationKey: "foo", AlertType: Info}, + `_e{2,4}:hi|okay|k:foo|t:info`, + }, { + &Event{Title: "hi", Text: "w/e", AlertType: Error, Priority: Normal}, + `_e{2,3}:hi|w/e|p:normal|t:error`, + }, { + &Event{Title: "hi", Text: "uh", Tags: []string{"host:foo", "app:bar"}}, + `_e{2,2}:hi|uh|#host:foo,app:bar`, + }, + } + + for _, m := range matrix { + r, err := m.event.Encode() + if err != nil { + t.Errorf("Error encoding: %s\n", err) + continue + } + if r != m.encoded { + t.Errorf("Expected `%s`, got `%s`\n", m.encoded, r) + } + } + + e := NewEvent("", "hi") + if _, err := e.Encode(); err == nil { + t.Errorf("Expected error on empty Title.") + } + + e = NewEvent("hi", "") + if _, err := e.Encode(); err == nil { + t.Errorf("Expected error on empty Text.") + } + + e = NewEvent("hello", "world") + s, err := e.Encode("tag1", "tag2") + if err != nil { + t.Error(err) + } + expected := "_e{5,5}:hello|world|#tag1,tag2" + if s != expected { + t.Errorf("Expected %s, got %s", expected, s) + } + if len(e.Tags) != 0 { + t.Errorf("Modified event in place illegally.") + } +} + +// These benchmarks show that using a buffer instead of sprintf-ing together +// a bunch of intermediate strings is 4-5x faster + +func BenchmarkFormatNew(b *testing.B) { + b.StopTimer() + c := &Client{} + c.Namespace = "foo.bar." + c.Tags = []string{"app:foo", "host:bar"} + b.StartTimer() + for i := 0; i < b.N; i++ { + c.format("system.cpu.idle", "10", []string{"foo"}, 1) + c.format("system.cpu.load", "0.1", nil, 0.9) + } +} + +// Old formatting function, added to client for tests +func (c *Client) formatOld(name, value string, tags []string, rate float64) string { + if rate < 1 { + value = fmt.Sprintf("%s|@%f", value, rate) + } + if c.Namespace != "" { + name = fmt.Sprintf("%s%s", c.Namespace, name) + } + + tags = append(c.Tags, tags...) + if len(tags) > 0 { + value = fmt.Sprintf("%s|#%s", value, strings.Join(tags, ",")) + } + + return fmt.Sprintf("%s:%s", name, value) + +} + +func BenchmarkFormatOld(b *testing.B) { + b.StopTimer() + c := &Client{} + c.Namespace = "foo.bar." + c.Tags = []string{"app:foo", "host:bar"} + b.StartTimer() + for i := 0; i < b.N; i++ { + c.formatOld("system.cpu.idle", "10", []string{"foo"}, 1) + c.formatOld("system.cpu.load", "0.1", nil, 0.9) + } +}