From a317cb9d5f69575bd85b9dac85b9baa83e8d7c7f Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 1 Mar 2014 15:53:27 -0800 Subject: [PATCH 001/217] Initial commit Also some philosophy. Because why not. --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..7d6629d --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +Goji +==== + +Goji is a minimalistic web framework inspired by Sinatra. + + +Philosophy +---------- + +Most of the design decisions in Goji can be traced back to the fundamental +philosopy that the Go standard library got things Mostly Right, and if it +didn't, it at least is good enough that it's not worth fighting. + +Therefore, Goji leans heavily on the standard library, and in particular its +interfaces and idioms. You can expect to be able to use most of Goji in exactly +the manner you would use a comparable standard library function, and have it +function in exactly the way you would expect. + +Also in this vein, Goji makes use of Go's `flag` package, and in particular the +default global flag set. Third party packages that have global state and squat +on global namespaces is something to be suspicious of, but the `flag` package is +also the closest thing Go has to a unified configuration API, and when used +tastefully it can make everyone's lives a bit easier. Wherever possible, the use +of these flags is opt-out, at the cost of additional complexity for the user. + +Goji also makes an attempt to not be magical -- explicit is better than +implicit. Goji does make use of reflection and `interface{}`, but only when an +API would be impossible or cumbersome without it. From f67c7d59eeac7ddd36dc88b24c0b251c268a70d2 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 1 Mar 2014 15:55:58 -0800 Subject: [PATCH 002/217] Parameter parsing Package param implements parameter parsing into a target struct (in much the same way as encoding/json parses JSON into a struct). It targets the common jQuery.param / Ruby on Rails style parameter serialization format. --- param/crazy_test.go | 56 +++++ param/error_helpers.go | 34 +++ param/param.go | 60 +++++ param/param_test.go | 505 +++++++++++++++++++++++++++++++++++++++++ param/parse.go | 214 +++++++++++++++++ param/pebkac_test.go | 58 +++++ param/struct.go | 117 ++++++++++ param/struct_test.go | 106 +++++++++ 8 files changed, 1150 insertions(+) create mode 100644 param/crazy_test.go create mode 100644 param/error_helpers.go create mode 100644 param/param.go create mode 100644 param/param_test.go create mode 100644 param/parse.go create mode 100644 param/pebkac_test.go create mode 100644 param/struct.go create mode 100644 param/struct_test.go diff --git a/param/crazy_test.go b/param/crazy_test.go new file mode 100644 index 0000000..46538cd --- /dev/null +++ b/param/crazy_test.go @@ -0,0 +1,56 @@ +package param + +import ( + "net/url" + "testing" +) + +type Crazy struct { + A *Crazy + B *Crazy + Value int + Slice []int + Map map[string]Crazy +} + +func TestCrazy(t *testing.T) { + t.Parallel() + + c := Crazy{} + err := Parse(url.Values{ + "A[B][B][A][Value]": {"1"}, + "B[A][A][Slice][]": {"3", "1", "4"}, + "B[Map][hello][A][Value]": {"8"}, + "A[Value]": {"2"}, + "A[Slice][]": {"9", "1", "1"}, + "Value": {"42"}, + }, &c) + if err != nil { + t.Error("Error parsing craziness: ", err) + } + + // Exhaustively checking everything here is going to be a huge pain, so + // let's just hope for the best, pretend NPEs don't exist, and hope that + // this test covers enough stuff that it's actually useful. + assertEqual(t, "c.A.B.B.A.Value", 1, c.A.B.B.A.Value) + assertEqual(t, "c.A.Value", 2, c.A.Value) + assertEqual(t, "c.Value", 42, c.Value) + assertEqual(t, `c.B.Map["hello"].A.Value`, 8, c.B.Map["hello"].A.Value) + + assertEqual(t, "c.A.B.B.B", (*Crazy)(nil), c.A.B.B.B) + assertEqual(t, "c.A.B.A", (*Crazy)(nil), c.A.B.A) + assertEqual(t, "c.A.A", (*Crazy)(nil), c.A.A) + + if c.Slice != nil || c.Map != nil { + t.Error("Map and Slice should not be set") + } + + sl := c.B.A.A.Slice + if len(sl) != 3 || sl[0] != 3 || sl[1] != 1 || sl[2] != 4 { + t.Error("Something is wrong with c.B.A.A.Slice") + } + sl = c.A.Slice + if len(sl) != 3 || sl[0] != 9 || sl[1] != 1 || sl[2] != 1 { + t.Error("Something is wrong with c.A.Slice") + } +} diff --git a/param/error_helpers.go b/param/error_helpers.go new file mode 100644 index 0000000..8033c9a --- /dev/null +++ b/param/error_helpers.go @@ -0,0 +1,34 @@ +package param + +import ( + "errors" + "fmt" + "log" +) + +// TODO: someday it might be nice to throw typed errors instead of weird strings + +// Testing log.Fatal in tests is... not a thing. Allow tests to stub it out. +var pebkacTesting bool + +const errPrefix = "param/parse: " +const yourFault = " This is a bug in your use of the param library." + +// Panic with a formatted error. The param library uses panics to quickly unwind +// the call stack and return a user error +func perr(format string, a ...interface{}) { + err := errors.New(errPrefix + fmt.Sprintf(format, a...)) + panic(err) +} + +// Problem exists between keyboard and chair. This function is used in cases of +// programmer error, i.e. an inappripriate use of the param library, to +// immediately force the program to halt with a hopefully helpful error message. +func pebkac(format string, a ...interface{}) { + err := errors.New(errPrefix + fmt.Sprintf(format, a...) + yourFault) + if pebkacTesting { + panic(err) + } else { + log.Fatal(err) + } +} diff --git a/param/param.go b/param/param.go new file mode 100644 index 0000000..0a1d8f5 --- /dev/null +++ b/param/param.go @@ -0,0 +1,60 @@ +/* +Package param deserializes parameter values into the given struct using magical +reflection ponies. Inspired by gorilla/schema, but uses Rails/jQuery style param +encoding instead of their weird dotted syntax. In particular, this package was +written with the intent of parsing the output of jQuery.param. + +This package uses struct tags to guess what names things ought to have. If a +struct value has a "param" tag defined, it will use that. If there is no "param" +tag defined, the name part of the "json" tag will be used. If that is not +defined, the name of the field itself will be used (no case transformation is +performed). + +If the name derived in this way is the string "-", param will refuse to set that +value. + +The parser is extremely strict, and will return an error if it has any +difficulty whatsoever in parsing any parameter, or if there is any kind of type +mismatch. +*/ +package param + +import ( + "net/url" + "reflect" + "strings" +) + +// Parse the given arguments into the the given pointer to a struct object. +func Parse(params url.Values, target interface{}) (err error) { + v := reflect.ValueOf(target) + + defer func() { + if r := recover(); r != nil { + var ok bool + err, ok = r.(error) + if !ok { + panic(err) + } + } + }() + + if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct { + pebkac("Target of param.Parse must be a pointer to a struct. "+ + "We instead were passed a %v", v.Type()) + } + + el := v.Elem() + t := el.Type() + cache := cacheStruct(t) + + for key, values := range params { + sk, keytail := key, "" + if i := strings.IndexRune(key, '['); i != -1 { + sk, keytail = sk[:i], sk[i:] + } + parseStructField(cache, key, sk, keytail, values, el) + } + + return nil +} diff --git a/param/param_test.go b/param/param_test.go new file mode 100644 index 0000000..48f5e42 --- /dev/null +++ b/param/param_test.go @@ -0,0 +1,505 @@ +package param + +import ( + "net/url" + "reflect" + "strings" + "testing" + "time" +) + +type Everything struct { + Bool bool + Int int + Uint uint + Float float64 + Map map[string]int + Slice []int + String string + Struct Sub + Time time.Time + + PBool *bool + PInt *int + PUint *uint + PFloat *float64 + PMap *map[string]int + PSlice *[]int + PString *string + PStruct *Sub + PTime *time.Time + + PPInt **int + + ABool MyBool + AInt MyInt + AUint MyUint + AFloat MyFloat + AMap MyMap + APtr MyPtr + ASlice MySlice + AString MyString +} + +type Sub struct { + A int + B int +} + +type MyBool bool +type MyInt int +type MyUint uint +type MyFloat float64 +type MyMap map[MyString]MyInt +type MyPtr *MyInt +type MySlice []MyInt +type MyString string + +var boolAnswers = map[string]bool{ + "true": true, + "false": false, + "0": false, + "1": true, + "on": true, + "": false, +} + +var testTimeString = "1996-12-19T16:39:57-08:00" +var testTime time.Time + +func init() { + testTime, _ = time.Parse(time.RFC3339, testTimeString) +} + +func singletonErrors(t *testing.T, field, valid, invalid string) { + e := Everything{} + + err := Parse(url.Values{field: {invalid}}, &e) + if err == nil { + t.Errorf("Expected error parsing %q as %s", invalid, field) + } + + err = Parse(url.Values{field + "[]": {valid}}, &e) + if err == nil { + t.Errorf("Expected error parsing nested %s", field) + } + + err = Parse(url.Values{field + "[nested]": {valid}}, &e) + if err == nil { + t.Errorf("Expected error parsing nested %s", field) + } + + err = Parse(url.Values{field: {valid, valid}}, &e) + if err == nil { + t.Errorf("Expected error passing %s twice", field) + } +} + +func TestBool(t *testing.T) { + t.Parallel() + + for val, correct := range boolAnswers { + e := Everything{} + e.Bool = !correct + err := Parse(url.Values{"Bool": {val}}, &e) + if err != nil { + t.Error("Parse error on key: ", val) + } + assertEqual(t, "e.Bool", correct, e.Bool) + } +} + +func TestBoolTyped(t *testing.T) { + t.Parallel() + + e := Everything{} + err := Parse(url.Values{"ABool": {"true"}}, &e) + if err != nil { + t.Error("Parse error for typed bool") + } + assertEqual(t, "e.ABool", MyBool(true), e.ABool) +} + +func TestBoolErrors(t *testing.T) { + t.Parallel() + singletonErrors(t, "Bool", "true", "llama") +} + +var intAnswers = map[string]int{ + "0": 0, + "9001": 9001, + "-42": -42, +} + +func TestInt(t *testing.T) { + t.Parallel() + + for val, correct := range intAnswers { + e := Everything{} + e.Int = 1 + err := Parse(url.Values{"Int": {val}}, &e) + if err != nil { + t.Error("Parse error on key: ", val) + } + assertEqual(t, "e.Int", correct, e.Int) + } +} + +func TestIntTyped(t *testing.T) { + t.Parallel() + + e := Everything{} + err := Parse(url.Values{"AInt": {"1"}}, &e) + if err != nil { + t.Error("Parse error for typed int") + } + assertEqual(t, "e.AInt", MyInt(1), e.AInt) +} + +func TestIntErrors(t *testing.T) { + t.Parallel() + singletonErrors(t, "Int", "1", "llama") + + e := Everything{} + err := Parse(url.Values{"Int": {"4.2"}}, &e) + if err == nil { + t.Error("Expected error parsing float as int") + } +} + +var uintAnswers = map[string]uint{ + "0": 0, + "9001": 9001, +} + +func TestUint(t *testing.T) { + t.Parallel() + + for val, correct := range uintAnswers { + e := Everything{} + e.Uint = 1 + err := Parse(url.Values{"Uint": {val}}, &e) + if err != nil { + t.Error("Parse error on key: ", val) + } + assertEqual(t, "e.Uint", correct, e.Uint) + } +} + +func TestUintTyped(t *testing.T) { + t.Parallel() + + e := Everything{} + err := Parse(url.Values{"AUint": {"1"}}, &e) + if err != nil { + t.Error("Parse error for typed uint") + } + assertEqual(t, "e.AUint", MyUint(1), e.AUint) +} + +func TestUintErrors(t *testing.T) { + t.Parallel() + singletonErrors(t, "Uint", "1", "llama") + + e := Everything{} + err := Parse(url.Values{"Uint": {"4.2"}}, &e) + if err == nil { + t.Error("Expected error parsing float as uint") + } + + err = Parse(url.Values{"Uint": {"-42"}}, &e) + if err == nil { + t.Error("Expected error parsing negative number as uint") + } +} + +var floatAnswers = map[string]float64{ + "0": 0, + "9001": 9001, + "-42": -42, + "9001.0": 9001.0, + "4.2": 4.2, + "-9.000001": -9.000001, +} + +func TestFloat(t *testing.T) { + t.Parallel() + + for val, correct := range floatAnswers { + e := Everything{} + e.Float = 1 + err := Parse(url.Values{"Float": {val}}, &e) + if err != nil { + t.Error("Parse error on key: ", val) + } + assertEqual(t, "e.Float", correct, e.Float) + } +} + +func TestFloatTyped(t *testing.T) { + t.Parallel() + + e := Everything{} + err := Parse(url.Values{"AFloat": {"1.0"}}, &e) + if err != nil { + t.Error("Parse error for typed float") + } + assertEqual(t, "e.AFloat", MyFloat(1.0), e.AFloat) +} + +func TestFloatErrors(t *testing.T) { + t.Parallel() + singletonErrors(t, "Float", "1.0", "llama") +} + +func TestMap(t *testing.T) { + t.Parallel() + e := Everything{} + + err := Parse(url.Values{ + "Map[one]": {"1"}, + "Map[two]": {"2"}, + "Map[three]": {"3"}, + }, &e) + if err != nil { + t.Error("Parse error in map: ", err) + } + + for k, v := range map[string]int{"one": 1, "two": 2, "three": 3} { + if mv, ok := e.Map[k]; !ok { + t.Errorf("Key %q not in map", k) + } else { + assertEqual(t, "Map["+k+"]", v, mv) + } + } +} + +func TestMapTyped(t *testing.T) { + t.Parallel() + + e := Everything{} + err := Parse(url.Values{"AMap[one]": {"1"}}, &e) + if err != nil { + t.Error("Parse error for typed map") + } + assertEqual(t, "e.AMap[one]", MyInt(1), e.AMap[MyString("one")]) +} + +func TestMapErrors(t *testing.T) { + t.Parallel() + e := Everything{} + + err := Parse(url.Values{"Map[]": {"llama"}}, &e) + if err == nil { + t.Error("expected error parsing empty map key") + } + + err = Parse(url.Values{"Map": {"llama"}}, &e) + if err == nil { + t.Error("expected error parsing map without key") + } + + err = Parse(url.Values{"Map[": {"llama"}}, &e) + if err == nil { + t.Error("expected error parsing map with malformed key") + } +} + +func testPtr(t *testing.T, key, in string, out interface{}) { + e := Everything{} + + err := Parse(url.Values{key: {in}}, &e) + if err != nil { + t.Errorf("Parse error while parsing pointer e.%s: %v", key, err) + } + fieldKey := key + if i := strings.IndexRune(fieldKey, '['); i >= 0 { + fieldKey = fieldKey[:i] + } + v := reflect.ValueOf(e).FieldByName(fieldKey) + if v.IsNil() { + t.Errorf("Expected param to allocate pointer for e.%s", key) + } else { + assertEqual(t, "*e."+key, out, v.Elem().Interface()) + } +} + +func TestPtr(t *testing.T) { + t.Parallel() + testPtr(t, "PBool", "true", true) + testPtr(t, "PInt", "2", 2) + testPtr(t, "PUint", "2", uint(2)) + testPtr(t, "PFloat", "2.0", 2.0) + testPtr(t, "PMap[llama]", "4", map[string]int{"llama": 4}) + testPtr(t, "PSlice[]", "4", []int{4}) + testPtr(t, "PString", "llama", "llama") + testPtr(t, "PStruct[B]", "2", Sub{0, 2}) + testPtr(t, "PTime", testTimeString, testTime) + + foo := 2 + testPtr(t, "PPInt", "2", &foo) +} + +func TestPtrTyped(t *testing.T) { + t.Parallel() + + e := Everything{} + err := Parse(url.Values{"APtr": {"1"}}, &e) + if err != nil { + t.Error("Parse error for typed pointer") + } + assertEqual(t, "e.APtr", MyInt(1), *e.APtr) +} + +func TestSlice(t *testing.T) { + t.Parallel() + + e := Everything{} + err := Parse(url.Values{"Slice[]": {"3", "1", "4"}}, &e) + if err != nil { + t.Error("Parse error for slice") + } + if e.Slice == nil { + t.Fatal("Expected param to allocate a slice") + } + if len(e.Slice) != 3 { + t.Fatal("Expected a slice of length 3") + } + + assertEqual(t, "e.Slice[0]", 3, e.Slice[0]) + assertEqual(t, "e.Slice[1]", 1, e.Slice[1]) + assertEqual(t, "e.Slice[2]", 4, e.Slice[2]) +} + +func TestSliceTyped(t *testing.T) { + t.Parallel() + e := Everything{} + err := Parse(url.Values{"ASlice[]": {"3", "1", "4"}}, &e) + if err != nil { + t.Error("Parse error for typed slice") + } + if e.ASlice == nil { + t.Fatal("Expected param to allocate a slice") + } + if len(e.ASlice) != 3 { + t.Fatal("Expected a slice of length 3") + } + + assertEqual(t, "e.ASlice[0]", MyInt(3), e.ASlice[0]) + assertEqual(t, "e.ASlice[1]", MyInt(1), e.ASlice[1]) + assertEqual(t, "e.ASlice[2]", MyInt(4), e.ASlice[2]) +} + +func TestSliceErrors(t *testing.T) { + t.Parallel() + e := Everything{} + err := Parse(url.Values{"Slice": {"1"}}, &e) + if err == nil { + t.Error("expected error parsing slice without key") + } + + err = Parse(url.Values{"Slice[llama]": {"1"}}, &e) + if err == nil { + t.Error("expected error parsing slice with string key") + } + + err = Parse(url.Values{"Slice[": {"1"}}, &e) + if err == nil { + t.Error("expected error parsing malformed slice key") + } +} + +var stringAnswer = "This is the world's best string" + +func TestString(t *testing.T) { + t.Parallel() + e := Everything{} + + err := Parse(url.Values{"String": {stringAnswer}}, &e) + if err != nil { + t.Error("Parse error in string: ", err) + } + + assertEqual(t, "e.String", stringAnswer, e.String) +} + +func TestStringTyped(t *testing.T) { + t.Parallel() + + e := Everything{} + err := Parse(url.Values{"AString": {"llama"}}, &e) + if err != nil { + t.Error("Parse error for typed string") + } + assertEqual(t, "e.AString", MyString("llama"), e.AString) +} + +func TestStruct(t *testing.T) { + t.Parallel() + e := Everything{} + + err := Parse(url.Values{ + "Struct[A]": {"1"}, + }, &e) + if err != nil { + t.Error("Parse error in struct: ", err) + } + assertEqual(t, "e.Struct.A", 1, e.Struct.A) + assertEqual(t, "e.Struct.B", 0, e.Struct.B) + + err = Parse(url.Values{ + "Struct[A]": {"4"}, + "Struct[B]": {"2"}, + }, &e) + if err != nil { + t.Error("Parse error in struct: ", err) + } + assertEqual(t, "e.Struct.A", 4, e.Struct.A) + assertEqual(t, "e.Struct.B", 2, e.Struct.B) +} + +func TestStructErrors(t *testing.T) { + t.Parallel() + e := Everything{} + + err := Parse(url.Values{"Struct[]": {"llama"}}, &e) + if err == nil { + t.Error("expected error parsing empty struct key") + } + + err = Parse(url.Values{"Struct": {"llama"}}, &e) + if err == nil { + t.Error("expected error parsing struct without key") + } + + err = Parse(url.Values{"Struct[": {"llama"}}, &e) + if err == nil { + t.Error("expected error parsing malformed struct key") + } + + err = Parse(url.Values{"Struct[C]": {"llama"}}, &e) + if err == nil { + t.Error("expected error parsing unknown") + } +} + +func TestTextUnmarshaler(t *testing.T) { + t.Parallel() + e := Everything{} + + err := Parse(url.Values{"Time": {testTimeString}}, &e) + if err != nil { + t.Error("parse error for TextUnmarshaler (Time): ", err) + } + assertEqual(t, "e.Time", testTime, e.Time) +} + +func TestTextUnmarshalerError(t *testing.T) { + t.Parallel() + e := Everything{} + + err := Parse(url.Values{"Time": {"llama"}}, &e) + if err == nil { + t.Error("expected error parsing llama as time") + } +} diff --git a/param/parse.go b/param/parse.go new file mode 100644 index 0000000..e59432a --- /dev/null +++ b/param/parse.go @@ -0,0 +1,214 @@ +package param + +import ( + "encoding" + "fmt" + "reflect" + "strconv" + "strings" +) + +var textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() + +// Generic parse dispatcher. This function's signature is the interface of all +// parse functions. `key` is the entire key that is currently being parsed, such +// as "foo[bar][]". `keytail` is the portion of the string that the current +// parser is responsible for, for instance "[bar][]". `values` is the list of +// values assigned to this key, and `target` is where the resulting typed value +// should be Set() to. +func parse(key, keytail string, values []string, target reflect.Value) { + t := target.Type() + if reflect.PtrTo(t).Implements(textUnmarshalerType) { + parseTextUnmarshaler(key, keytail, values, target) + return + } + + switch k := target.Kind(); k { + case reflect.Bool: + parseBool(key, keytail, values, target) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + parseInt(key, keytail, values, target) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + parseUint(key, keytail, values, target) + case reflect.Float32, reflect.Float64: + parseFloat(key, keytail, values, target) + case reflect.Map: + parseMap(key, keytail, values, target) + case reflect.Ptr: + parsePtr(key, keytail, values, target) + case reflect.Slice: + parseSlice(key, keytail, values, target) + case reflect.String: + parseString(key, keytail, values, target) + case reflect.Struct: + parseStruct(key, keytail, values, target) + + default: + pebkac("unsupported object of type %v and kind %v.", + target.Type(), k) + } +} + +// We pass down both the full key ("foo[bar][]") and the part the current layer +// is responsible for making sense of ("[bar][]"). This computes the other thing +// you probably want to know, which is the path you took to get here ("foo"). +func kpath(key, keytail string) string { + l, t := len(key), len(keytail) + return key[:l-t] +} + +// Helper for validating that a value has been passed exactly once, and that the +// user is not attempting to nest on the key. +func primitive(tipe, key, keytail string, values []string) { + if keytail != "" { + perr("expected %s for key %q, got nested value", tipe, + kpath(key, keytail)) + } + if len(values) != 1 { + perr("expected %s for key %q, but key passed %v times", tipe, + kpath(key, keytail), len(values)) + } +} + +func keyed(tipe reflect.Type, key, keytail string) (string, string) { + idx := strings.IndexRune(keytail, ']') + // Keys must be at least 1 rune wide: we refuse to use the empty string + // as the key + if len(keytail) < 3 || keytail[0] != '[' || idx < 2 { + perr("expected a square bracket delimited index for %q "+ + "(of type %v)", kpath(key, keytail), tipe) + } + return keytail[1:idx], keytail[idx+1:] +} + +func parseTextUnmarshaler(key, keytail string, values []string, target reflect.Value) { + primitive("encoding.TextUnmarshaler", key, keytail, values) + + tu := target.Addr().Interface().(encoding.TextUnmarshaler) + err := tu.UnmarshalText([]byte(values[0])) + if err != nil { + perr("error while calling UnmarshalText on %v for key %q: %v", + target.Type(), kpath(key, keytail), err) + } +} + +func parseBool(key, keytail string, values []string, target reflect.Value) { + primitive("bool", key, keytail, values) + + switch values[0] { + case "true", "1", "on": + target.SetBool(true) + case "false", "0", "": + target.SetBool(false) + default: + perr("could not parse key %q as bool", kpath(key, keytail)) + } +} + +func parseInt(key, keytail string, values []string, target reflect.Value) { + primitive("int", key, keytail, values) + + t := target.Type() + i, err := strconv.ParseInt(values[0], 10, t.Bits()) + if err != nil { + perr("error parsing key %q as int: %v", kpath(key, keytail), + err) + } + target.SetInt(i) +} + +func parseUint(key, keytail string, values []string, target reflect.Value) { + primitive("uint", key, keytail, values) + + t := target.Type() + i, err := strconv.ParseUint(values[0], 10, t.Bits()) + if err != nil { + perr("error parsing key %q as uint: %v", kpath(key, keytail), + err) + } + target.SetUint(i) +} + +func parseFloat(key, keytail string, values []string, target reflect.Value) { + primitive("float", key, keytail, values) + + t := target.Type() + f, err := strconv.ParseFloat(values[0], t.Bits()) + if err != nil { + perr("error parsing key %q as float: %v", kpath(key, keytail), + err) + } + + target.SetFloat(f) +} + +func parseString(key, keytail string, values []string, target reflect.Value) { + primitive("string", key, keytail, values) + + target.SetString(values[0]) +} + +func parseSlice(key, keytail string, values []string, target reflect.Value) { + // BUG(carl): We currently do not handle slices of nested types. If + // support is needed, the implementation probably could be fleshed out. + if keytail != "[]" { + perr("unexpected array nesting for key %q: %q", + kpath(key, keytail), keytail) + } + t := target.Type() + + slice := reflect.MakeSlice(t, len(values), len(values)) + kp := kpath(key, keytail) + for i, _ := range values { + // We actually cheat a little bit and modify the key so we can + // generate better debugging messages later + key := fmt.Sprintf("%s[%d]", kp, i) + parse(key, "", values[i:i+1], slice.Index(i)) + } + target.Set(slice) +} + +func parseMap(key, keytail string, values []string, target reflect.Value) { + t := target.Type() + mapkey, maptail := keyed(t, key, keytail) + + // BUG(carl): We don't support any map keys except strings, although + // there's no reason we shouldn't be able to throw the value through our + // unparsing stack. + var mk reflect.Value + if t.Key().Kind() == reflect.String { + mk = reflect.ValueOf(mapkey).Convert(t.Key()) + } else { + pebkac("key for map %v isn't a string (it's a %v).", t, t.Key()) + } + + if target.IsNil() { + target.Set(reflect.MakeMap(t)) + } + + val := target.MapIndex(mk) + if !val.IsValid() || !val.CanSet() { + // It's a teensy bit annoying that the value returned by + // MapIndex isn't Set()table if the key exists. + val = reflect.New(t.Elem()).Elem() + } + parse(key, maptail, values, val) + target.SetMapIndex(mk, val) +} + +func parseStruct(key, keytail string, values []string, target reflect.Value) { + t := target.Type() + sk, skt := keyed(t, key, keytail) + cache := cacheStruct(t) + + parseStructField(cache, key, sk, skt, values, target) +} + +func parsePtr(key, keytail string, values []string, target reflect.Value) { + t := target.Type() + + if target.IsNil() { + target.Set(reflect.New(t.Elem())) + } + parse(key, keytail, values, target.Elem()) +} diff --git a/param/pebkac_test.go b/param/pebkac_test.go new file mode 100644 index 0000000..71d64eb --- /dev/null +++ b/param/pebkac_test.go @@ -0,0 +1,58 @@ +package param + +import ( + "net/url" + "strings" + "testing" +) + +type Bad struct { + Unknown interface{} +} + +type Bad2 struct { + Unknown *interface{} +} + +type Bad3 struct { + BadMap map[int]int +} + +// These tests are not parallel so we can frob pebkac behavior in an isolated +// way + +func assertPebkac(t *testing.T, err error) { + if err == nil { + t.Error("Expected PEBKAC error message") + } else if !strings.HasSuffix(err.Error(), yourFault) { + t.Errorf("Expected PEBKAC error, but got: %v", err) + } +} + +func TestBadInputs(t *testing.T) { + pebkacTesting = true + + err := Parse(url.Values{"Unknown": {"4"}}, Bad{}) + assertPebkac(t, err) + + b := &Bad{} + err = Parse(url.Values{"Unknown": {"4"}}, &b) + assertPebkac(t, err) + + pebkacTesting = false +} + +func TestBadTypes(t *testing.T) { + pebkacTesting = true + + err := Parse(url.Values{"Unknown": {"4"}}, &Bad{}) + assertPebkac(t, err) + + err = Parse(url.Values{"Unknown": {"4"}}, &Bad2{}) + assertPebkac(t, err) + + err = Parse(url.Values{"BadMap[llama]": {"4"}}, &Bad3{}) + assertPebkac(t, err) + + pebkacTesting = false +} diff --git a/param/struct.go b/param/struct.go new file mode 100644 index 0000000..314d5b0 --- /dev/null +++ b/param/struct.go @@ -0,0 +1,117 @@ +package param + +import ( + "reflect" + "strings" + "sync" +) + +// We decode a lot of structs (since it's the top-level thing this library +// decodes) and it takes a fair bit of work to reflect upon the struct to figure +// out what we want to do. Instead of doing this on every invocation, we cache +// metadata about each struct the first time we see it. The upshot is that we +// save some work every time. The downside is we are forced to briefly acquire +// a lock to access the cache in a thread-safe way. If this ever becomes a +// bottleneck, both the lock and the cache can be sharded or something. +type structCache map[string]cacheLine +type cacheLine struct { + offset int + parse func(string, string, []string, reflect.Value) +} + +var cacheLock sync.RWMutex +var cache = make(map[reflect.Type]structCache) + +func cacheStruct(t reflect.Type) structCache { + cacheLock.RLock() + sc, ok := cache[t] + cacheLock.RUnlock() + + if ok { + return sc + } + + // It's okay if two people build struct caches simultaneously + sc = make(structCache) + for i := 0; i < t.NumField(); i++ { + sf := t.Field(i) + // Only unexported fields have a PkgPath; we want to only cache + // exported fields. + if sf.PkgPath != "" { + continue + } + name := extractName(sf) + if name != "-" { + sc[name] = cacheLine{i, extractHandler(t, sf)} + } + } + + cacheLock.Lock() + cache[t] = sc + cacheLock.Unlock() + + return sc +} + +// Extract the name of the given struct field, looking at struct tags as +// appropriate. +func extractName(sf reflect.StructField) string { + name := sf.Tag.Get("param") + if name == "" { + name = sf.Tag.Get("json") + idx := strings.IndexRune(name, ',') + if idx >= 0 { + name = name[:idx] + } + } + if name == "" { + name = sf.Name + } + + return name +} + +func extractHandler(s reflect.Type, sf reflect.StructField) func(string, string, []string, reflect.Value) { + if reflect.PtrTo(sf.Type).Implements(textUnmarshalerType) { + return parseTextUnmarshaler + } + + switch sf.Type.Kind() { + case reflect.Bool: + return parseBool + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return parseInt + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return parseUint + case reflect.Float32, reflect.Float64: + return parseFloat + case reflect.Map: + return parseMap + case reflect.Ptr: + return parsePtr + case reflect.Slice: + return parseSlice + case reflect.String: + return parseString + case reflect.Struct: + return parseStruct + + default: + pebkac("struct %v has illegal field %q (type %v, kind %v).", + s, sf.Name, sf.Type, sf.Type.Kind()) + return nil + } +} + +// We have to parse two types of structs: ones at the top level, whose keys +// don't have square brackets around them, and nested structs, which do. +func parseStructField(cache structCache, key, sk, keytail string, values []string, target reflect.Value) { + l, ok := cache[sk] + if !ok { + perr("unknown key %q for struct at key %q", sk, + kpath(key, keytail)) + } + f := target.Field(l.offset) + + l.parse(key, keytail, values, f) +} diff --git a/param/struct_test.go b/param/struct_test.go new file mode 100644 index 0000000..ecba3e2 --- /dev/null +++ b/param/struct_test.go @@ -0,0 +1,106 @@ +package param + +import ( + "reflect" + "testing" +) + +type Fruity struct { + A bool + B int `json:"banana"` + C uint `param:"cherry"` + D float64 `json:"durian" param:"dragonfruit"` + E int `json:"elderberry" param:"-"` + F map[string]int `json:"-" param:"fig"` + G *int `json:"grape,omitempty"` + H []int `param:"honeydew" json:"huckleberry"` + I string `foobar:"iyokan"` + J Cheesy `param:"jackfruit" cheese:"jarlsberg"` +} + +type Cheesy struct { + A int `param:"affinois"` + B int `param:"brie"` + C int `param:"camembert"` + D int `param:"delice d'argental"` +} + +type Private struct { + Public, private int +} + +var fruityType = reflect.TypeOf(Fruity{}) +var cheesyType = reflect.TypeOf(Cheesy{}) +var privateType = reflect.TypeOf(Private{}) + +var fruityNames = []string{ + "A", "banana", "cherry", "dragonfruit", "-", "fig", "grape", "honeydew", + "I", "jackfruit", +} + +var fruityCache = map[string]cacheLine{ + "A": {0, parseBool}, + "banana": {1, parseInt}, + "cherry": {2, parseUint}, + "dragonfruit": {3, parseFloat}, + "fig": {5, parseMap}, + "grape": {6, parsePtr}, + "honeydew": {7, parseSlice}, + "I": {8, parseString}, + "jackfruit": {9, parseStruct}, +} + +func assertEqual(t *testing.T, what string, e, a interface{}) { + if !reflect.DeepEqual(e, a) { + t.Errorf("Expected %s to be %v, was actually %v", what, e, a) + } +} + +func TestNames(t *testing.T) { + t.Parallel() + + for i, val := range fruityNames { + name := extractName(fruityType.Field(i)) + assertEqual(t, "tag", val, name) + } +} + +func TestCacheStruct(t *testing.T) { + t.Parallel() + + sc := cacheStruct(fruityType) + + if len(sc) != len(fruityCache) { + t.Errorf("Cache has %d keys, but expected %d", len(sc), + len(fruityCache)) + } + + for k, v := range fruityCache { + sck, ok := sc[k] + if !ok { + t.Errorf("Could not find key %q in cache", k) + continue + } + if sck.offset != v.offset { + t.Errorf("Cache for %q: expected offset %d but got %d", + k, sck.offset, v.offset) + } + // We want to compare function pointer equality, and this + // appears to be the only way + a := reflect.ValueOf(sck.parse) + b := reflect.ValueOf(v.parse) + if a.Pointer() != b.Pointer() { + t.Errorf("Parse mismatch for %q: %v, expected %v", k, a, + b) + } + } +} + +func TestPrivate(t *testing.T) { + t.Parallel() + + sc := cacheStruct(privateType) + if len(sc) != 1 { + t.Error("Expected Private{} to have one cachable field") + } +} From a79303168df73cbb9c76396b539c17a2d677df3c Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 2 Mar 2014 16:49:34 -0800 Subject: [PATCH 003/217] Socket bind helper package Package bind provides a convenient syntax for binding to sockets, as well as a fair bit of magic to automatically Do The Right Thing in both development and production. --- bind/bind.go | 115 ++++++++++++++++++++++++++++++++++++++++++++++++ bind/einhorn.go | 88 ++++++++++++++++++++++++++++++++++++ bind/systemd.go | 34 ++++++++++++++ 3 files changed, 237 insertions(+) create mode 100644 bind/bind.go create mode 100644 bind/einhorn.go create mode 100644 bind/systemd.go diff --git a/bind/bind.go b/bind/bind.go new file mode 100644 index 0000000..5541ec9 --- /dev/null +++ b/bind/bind.go @@ -0,0 +1,115 @@ +/* +Package bind provides a convenient way to bind to sockets. It exposes a flag in +the default flag set named "bind" which provides syntax to bind TCP and UNIX +sockets. It also supports binding to arbitrary file descriptors passed by a +parent (for instance, systemd), and for binding to Einhorn sockets (including +Einhorn ACK support). + +If the value passed to bind contains a colon, as in ":8000" or "127.0.0.1:9001", +it will be treated as a TCP address. If it begins with a "/" or a ".", it will +be treated as a path to a UNIX socket. If it begins with the string "fd@", as in +"fd@3", it will be treated as a file descriptor (useful for use with systemd, +for instance). If it begins with the string "einhorn@", as in "einhorn@0", the +corresponding einhorn socket will be used. + +If an option is not explicitly passed, the implementation will automatically +select between using "einhorn@0", "fd@3", and ":8000", depending on whether +Einhorn or systemd (or neither) is detected. + +This package is a teensy bit magical, and goes out of its way to Do The Right +Thing in many situations, including in both development and production. If +you're looking for something less magical, you'd probably be better off just +calling net.Listen() the old-fashioned way. +*/ +package bind + +import ( + "flag" + "fmt" + "log" + "net" + "os" + "strconv" + "strings" + "sync" +) + +var bind string + +func init() { + einhornInit() + systemdInit() + + defaultBind := ":8000" + if usingEinhorn() { + defaultBind = "einhorn@0" + } else if usingSystemd() { + defaultBind = "fd@3" + } + flag.StringVar(&bind, "bind", defaultBind, + `Address to bind on. If this value has a colon, as in ":8000" or + "127.0.0.1:9001", it will be treated as a TCP address. If it + begins with a "/" or a ".", it will be treated as a path to a + UNIX socket. If it begins with the string "fd@", as in "fd@3", + it will be treated as a file descriptor (useful for use with + systemd, for instance). If it begins with the string "einhorn@", + as in "einhorn@0", the corresponding einhorn socket will be + used. If an option is not explicitly passed, the implementation + will automatically select among "einhorn@0" (Einhorn), "fd@3" + (systemd), and ":8000" (fallback) based on its environment.`) +} + +func listenTo(bind string) (net.Listener, error) { + if strings.Contains(bind, ":") { + return net.Listen("tcp", bind) + } else if strings.HasPrefix(bind, ".") || strings.HasPrefix(bind, "/") { + return net.Listen("unix", bind) + } else if strings.HasPrefix(bind, "fd@") { + fd, err := strconv.Atoi(bind[3:]) + if err != nil { + return nil, fmt.Errorf("Error while parsing fd %v: %v", + bind, err) + } + f := os.NewFile(uintptr(fd), bind) + return net.FileListener(f) + } else if strings.HasPrefix(bind, "einhorn@") { + fd, err := strconv.Atoi(bind[8:]) + if err != nil { + return nil, fmt.Errorf( + "Error while parsing einhorn %v: %v", bind, err) + } + return einhornBind(fd) + } + + return nil, fmt.Errorf("Error while parsing bind arg %v", bind) +} + +// Parse and bind to the specified address. If Socket encounters an error while +// parsing or binding to the given socket it will exit by calling log.Fatal. +func Socket(bind string) net.Listener { + l, err := listenTo(bind) + if err != nil { + log.Fatal(err) + } + return l +} + +// Parse and bind to the default socket as given to us by the flag module. If +// there was an error parsing or binding to that socket, Default will exit by +// calling `log.Fatal`. +func Default() net.Listener { + return Socket(bind) +} + +// I'm not sure why you'd ever want to call Ready() more than once, but we may +// as well be safe against it... +var ready sync.Once + +// Notify the environment (for now, just Einhorn) that the process is ready to +// receive traffic. Should be called at the last possible moment to maximize the +// chances that a faulty process exits before signaling that it's ready. +func Ready() { + ready.Do(func() { + einhornAck() + }) +} diff --git a/bind/einhorn.go b/bind/einhorn.go new file mode 100644 index 0000000..f756fb0 --- /dev/null +++ b/bind/einhorn.go @@ -0,0 +1,88 @@ +package bind + +import ( + "fmt" + "log" + "net" + "os" + "strconv" + "syscall" +) + +const tooBigErr = "bind: einhorn@%d not found (einhorn only passed %d fds)" +const bindErr = "bind: could not bind einhorn@%d: not running under einhorn" +const einhornErr = "bind: einhorn environment initialization error" +const ackErr = "bind: error ACKing to einhorn: %v" + +var einhornNumFds int + +func envInt(val string) (int, error) { + return strconv.Atoi(os.Getenv(val)) +} + +// Unfortunately this can't be a normal init function, because their execution +// order is undefined, and we need to run before the init() in bind.go. +func einhornInit() { + mpid, err := envInt("EINHORN_MASTER_PID") + if err != nil || mpid != os.Getppid() { + return + } + + einhornNumFds, err = envInt("EINHORN_FD_COUNT") + if err != nil { + einhornNumFds = 0 + return + } + + // Prevent einhorn's fds from leaking to our children + for i := 0; i < einhornNumFds; i++ { + fd := int(einhornFd(i).Fd()) + syscall.CloseOnExec(fd) + } +} + +func usingEinhorn() bool { + return einhornNumFds > 0 +} + +func einhornFd(n int) *os.File { + name := fmt.Sprintf("EINHORN_FD_%d", n) + fno, err := envInt(name) + if err != nil { + log.Fatal(einhornErr) + } + return os.NewFile(uintptr(fno), name) +} + +func einhornBind(n int) (net.Listener, error) { + if !usingEinhorn() { + return nil, fmt.Errorf(bindErr, n) + } + if n >= einhornNumFds || n < 0 { + return nil, fmt.Errorf(tooBigErr, n, einhornNumFds) + } + + f := einhornFd(n) + return net.FileListener(f) +} + +// Fun story: this is actually YAML, not JSON. +const ackMsg = `{"command":"worker:ack","pid":%d}` + "\n" + +func einhornAck() { + if !usingEinhorn() { + return + } + log.Print("bind: ACKing to einhorn") + + ctl, err := net.Dial("unix", os.Getenv("EINHORN_SOCK_PATH")) + if err != nil { + log.Fatalf(ackErr, err) + } + defer ctl.Close() + + _, err = fmt.Fprintf(ctl, ackMsg, os.Getpid()) + if err != nil { + log.Fatalf(ackErr, err) + } +} diff --git a/bind/systemd.go b/bind/systemd.go new file mode 100644 index 0000000..1648bc9 --- /dev/null +++ b/bind/systemd.go @@ -0,0 +1,34 @@ +package bind + +import ( + "os" + "syscall" +) + +const systemdMinFd = 3 + +var systemdNumFds int + +// Unfortunately this can't be a normal init function, because their execution +// order is undefined, and we need to run before the init() in bind.go. +func systemdInit() { + pid, err := envInt("LISTEN_PID") + if err != nil || pid != os.Getpid() { + return + } + + systemdNumFds, err = envInt("LISTEN_FDS") + if err != nil { + systemdNumFds = 0 + return + } + + // Prevent fds from leaking to our children + for i := 0; i < systemdNumFds; i++ { + syscall.CloseOnExec(systemdMinFd + i) + } +} + +func usingSystemd() bool { + return systemdNumFds > 0 +} From 96b81e193075cb30401430b0705b65d178b68f1d Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 8 Mar 2014 01:58:30 -0800 Subject: [PATCH 004/217] Graceful shutdown package Package graceful provides graceful shutdown support for net/http servers, net.Listeners and net.Conns. It does this through terrible, terrible hacks, but "oh well!" --- graceful/conn_test.go | 63 ++++++++++++ graceful/einhorn.go | 22 ++++ graceful/graceful.go | 136 +++++++++++++++++++++++++ graceful/middleware.go | 106 +++++++++++++++++++ graceful/middleware_test.go | 68 +++++++++++++ graceful/net.go | 198 ++++++++++++++++++++++++++++++++++++ graceful/net_test.go | 198 ++++++++++++++++++++++++++++++++++++ graceful/signal.go | 117 +++++++++++++++++++++ 8 files changed, 908 insertions(+) create mode 100644 graceful/conn_test.go create mode 100644 graceful/einhorn.go create mode 100644 graceful/graceful.go create mode 100644 graceful/middleware.go create mode 100644 graceful/middleware_test.go create mode 100644 graceful/net.go create mode 100644 graceful/net_test.go create mode 100644 graceful/signal.go diff --git a/graceful/conn_test.go b/graceful/conn_test.go new file mode 100644 index 0000000..d86f12e --- /dev/null +++ b/graceful/conn_test.go @@ -0,0 +1,63 @@ +package graceful + +import ( + "net" + "time" +) + +// Stub out a net.Conn. This is going to be painful. + +type fakeAddr struct{} + +func (f fakeAddr) Network() string { + return "fake" +} +func (f fakeAddr) String() string { + return "fake" +} + +type fakeConn struct { + onRead, onWrite, onClose, onLocalAddr, onRemoteAddr func() + onSetDeadline, onSetReadDeadline, onSetWriteDeadline func() +} + +// Here's my number, so... +func callMeMaybe(f func()) { + // I apologize for nothing. + if f != nil { + f() + } +} + +func (f fakeConn) Read(b []byte) (int, error) { + callMeMaybe(f.onRead) + return len(b), nil +} +func (f fakeConn) Write(b []byte) (int, error) { + callMeMaybe(f.onWrite) + return len(b), nil +} +func (f fakeConn) Close() error { + callMeMaybe(f.onClose) + return nil +} +func (f fakeConn) LocalAddr() net.Addr { + callMeMaybe(f.onLocalAddr) + return fakeAddr{} +} +func (f fakeConn) RemoteAddr() net.Addr { + callMeMaybe(f.onRemoteAddr) + return fakeAddr{} +} +func (f fakeConn) SetDeadline(t time.Time) error { + callMeMaybe(f.onSetDeadline) + return nil +} +func (f fakeConn) SetReadDeadline(t time.Time) error { + callMeMaybe(f.onSetReadDeadline) + return nil +} +func (f fakeConn) SetWriteDeadline(t time.Time) error { + callMeMaybe(f.onSetWriteDeadline) + return nil +} diff --git a/graceful/einhorn.go b/graceful/einhorn.go new file mode 100644 index 0000000..c8e7af2 --- /dev/null +++ b/graceful/einhorn.go @@ -0,0 +1,22 @@ +package graceful + +import ( + "log" + "os" + "strconv" + "syscall" +) + +func init() { + // This is a little unfortunate: goji/bind already knows whether we're + // running under einhorn, but we don't want to introduce a dependency + // between the two packages. Since the check is short enough, inlining + // it here seems "fine." + mpid, err := strconv.Atoi(os.Getenv("EINHORN_MASTER_PID")) + if err != nil || mpid != os.Getppid() { + return + } + + log.Print("graceful: Einhorn detected, adding SIGUSR2 handler") + AddSignal(syscall.SIGUSR2) +} diff --git a/graceful/graceful.go b/graceful/graceful.go new file mode 100644 index 0000000..121e307 --- /dev/null +++ b/graceful/graceful.go @@ -0,0 +1,136 @@ +/* +Package graceful implements graceful shutdown for HTTP servers by closing idle +connections after receiving a signal. By default, this package listens for +interrupts (i.e., SIGINT), but when it detects that it is running under Einhorn +it will additionally listen for SIGUSR2 as well, giving your application +automatic support for graceful upgrades. + +It's worth mentioning explicitly that this package is a hack to shim graceful +shutdown behavior into the net/http package provided in Go 1.2. It was written +by carefully reading the sequence of function calls net/http happened to use as +of this writing and finding enough surface area with which to add appropriate +behavior. There's a very good chance that this package will cease to work in +future versions of Go, but with any luck the standard library will add support +of its own by then. + +If you're interested in figuring out how this package works, we suggest you read +the documentation for WrapConn() and net.go. +*/ +package graceful + +import ( + "crypto/tls" + "net" + "net/http" + "time" +) + +// Exactly like net/http's Server. In fact, it *is* a net/http Server, just with +// different method implementations +type Server http.Server + +// About 200 years, also known as "forever" +const forever time.Duration = 200 * 365 * 24 * time.Hour + +/* +You might notice that these methods look awfully similar to the methods of the +same name from the go standard library--that's because they were stolen from +there! If go were more like, say, Ruby, it'd actually be possible to shim just +the Serve() method, since we can do everything we want from there. However, it's +not possible to get the other methods which call Serve() (ListenAndServe(), say) +to call your shimmed copy--they always call the original. + +Since I couldn't come up with a better idea, I just copy-and-pasted both +ListenAndServe and ListenAndServeTLS here more-or-less verbatim. "Oh well!" +*/ + +// Behaves exactly like the net/http function of the same name. +func (srv *Server) Serve(l net.Listener) (err error) { + go func() { + <-kill + l.Close() + }() + l = WrapListener(l) + + // Spawn a shadow http.Server to do the actual servering. We do this + // because we need to sketch on some of the parameters you passed in, + // and it's nice to keep our sketching to ourselves. + shadow := *(*http.Server)(srv) + + if shadow.ReadTimeout == 0 { + shadow.ReadTimeout = forever + } + shadow.Handler = Middleware(shadow.Handler) + + err = shadow.Serve(l) + + // We expect an error when we close the listener, so we indiscriminately + // swallow Serve errors when we're in a shutdown state. + select { + case <-kill: + return nil + default: + return err + } +} + +// Behaves exactly like the net/http function of the same name. +func (srv *Server) ListenAndServe() error { + addr := srv.Addr + if addr == "" { + addr = ":http" + } + l, e := net.Listen("tcp", addr) + if e != nil { + return e + } + return srv.Serve(l) +} + +// Behaves exactly like the net/http function of the same name. +func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { + addr := srv.Addr + if addr == "" { + addr = ":https" + } + config := &tls.Config{} + if srv.TLSConfig != nil { + *config = *srv.TLSConfig + } + if config.NextProtos == nil { + config.NextProtos = []string{"http/1.1"} + } + + var err error + config.Certificates = make([]tls.Certificate, 1) + config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return err + } + + conn, err := net.Listen("tcp", addr) + if err != nil { + return err + } + + tlsListener := tls.NewListener(conn, config) + return srv.Serve(tlsListener) +} + +// Behaves exactly like the net/http function of the same name. +func ListenAndServe(addr string, handler http.Handler) error { + server := &Server{Addr: addr, Handler: handler} + return server.ListenAndServe() +} + +// Behaves exactly like the net/http function of the same name. +func ListenAndServeTLS(addr, certfile, keyfile string, handler http.Handler) error { + server := &Server{Addr: addr, Handler: handler} + return server.ListenAndServeTLS(certfile, keyfile) +} + +// Behaves exactly like the net/http function of the same name. +func Serve(l net.Listener, handler http.Handler) error { + server := &Server{Handler: handler} + return server.Serve(l) +} diff --git a/graceful/middleware.go b/graceful/middleware.go new file mode 100644 index 0000000..34a3368 --- /dev/null +++ b/graceful/middleware.go @@ -0,0 +1,106 @@ +package graceful + +import ( + "bufio" + "net" + "net/http" +) + +/* +Graceful shutdown middleware. When a graceful shutdown is in progress, this +middleware intercepts responses to add a "Connection: close" header to politely +inform the client that we are about to go away. + +This package creates a shim http.ResponseWriter that it passes to subsequent +handlers. Unfortunately, there's a great many optional interfaces that this +http.ResponseWriter might implement (e.g., http.CloseNotifier, http.Flusher, and +http.Hijacker), and in order to perfectly proxy all of these options we'd be +left with some kind of awful powerset of ResponseWriters, and that's not even +counting all the other custom interfaces you might be expecting. Instead of +doing that, we have implemented two kinds of proxies: one that contains no +additional methods (i.e., exactly corresponding to the http.ResponseWriter +interface), and one that supports all three of http.CloseNotifier, http.Flusher, +and http.Hijacker. If you find that this is not enough, the original +http.ResponseWriter can be retrieved by calling Unwrap() on the proxy object. + +This middleware is automatically applied to every http.Handler passed to this +package, and most users will not need to call this function directly. It is +exported primarily for documentation purposes and in the off chance that someone +really wants more control over their http.Server than we currently provide. +*/ +func Middleware(h http.Handler) http.Handler { + if h == nil { + return nil + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, cn := w.(http.CloseNotifier) + _, fl := w.(http.Flusher) + _, hj := w.(http.Hijacker) + + bw := basicWriter{ResponseWriter: w} + + if cn && fl && hj { + h.ServeHTTP(&fancyWriter{bw}, r) + } else { + h.ServeHTTP(&bw, r) + } + if !bw.headerWritten { + bw.maybeClose() + } + }) +} + +type basicWriter struct { + http.ResponseWriter + headerWritten bool +} + +func (b *basicWriter) maybeClose() { + b.headerWritten = true + select { + case <-kill: + b.ResponseWriter.Header().Add("Connection", "close") + default: + } +} + +func (b *basicWriter) WriteHeader(code int) { + b.maybeClose() + b.ResponseWriter.WriteHeader(code) +} + +func (b *basicWriter) Write(buf []byte) (int, error) { + if !b.headerWritten { + b.maybeClose() + } + return b.ResponseWriter.Write(buf) +} + +func (b *basicWriter) Unwrap() http.ResponseWriter { + return b.ResponseWriter +} + +// Optimize for the common case of a ResponseWriter that supports all three of +// CloseNotifier, Flusher, and Hijacker. +type fancyWriter struct { + basicWriter +} + +func (f *fancyWriter) CloseNotify() <-chan bool { + cn := f.basicWriter.ResponseWriter.(http.CloseNotifier) + return cn.CloseNotify() +} +func (f *fancyWriter) Flush() { + fl := f.basicWriter.ResponseWriter.(http.Flusher) + fl.Flush() +} +func (f *fancyWriter) Hijack() (c net.Conn, b *bufio.ReadWriter, e error) { + hj := f.basicWriter.ResponseWriter.(http.Hijacker) + c, b, e = hj.Hijack() + + if conn, ok := c.(hijackConn); ok { + c = conn.hijack() + } + + return +} diff --git a/graceful/middleware_test.go b/graceful/middleware_test.go new file mode 100644 index 0000000..ecec606 --- /dev/null +++ b/graceful/middleware_test.go @@ -0,0 +1,68 @@ +package graceful + +import ( + "net/http" + "testing" +) + +type fakeWriter http.Header + +func (f fakeWriter) Header() http.Header { + return http.Header(f) +} +func (f fakeWriter) Write(buf []byte) (int, error) { + return len(buf), nil +} +func (f fakeWriter) WriteHeader(status int) {} + +func testClose(t *testing.T, h http.Handler, expectClose bool) { + m := Middleware(h) + r, _ := http.NewRequest("GET", "/", nil) + w := make(fakeWriter) + m.ServeHTTP(w, r) + + c, ok := w["Connection"] + if expectClose { + if !ok || len(c) != 1 || c[0] != "close" { + t.Fatal("Expected 'Connection: close'") + } + } else { + if ok { + t.Fatal("Did not expect Connection header") + } + } +} + +func TestNormal(t *testing.T) { + kill = make(chan struct{}) + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte{}) + }) + testClose(t, h, false) +} + +func TestClose(t *testing.T) { + kill = make(chan struct{}) + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(kill) + }) + testClose(t, h, true) +} + +func TestCloseWriteHeader(t *testing.T) { + kill = make(chan struct{}) + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(kill) + w.WriteHeader(200) + }) + testClose(t, h, true) +} + +func TestCloseWrite(t *testing.T) { + kill = make(chan struct{}) + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(kill) + w.Write([]byte{}) + }) + testClose(t, h, true) +} diff --git a/graceful/net.go b/graceful/net.go new file mode 100644 index 0000000..b86af8c --- /dev/null +++ b/graceful/net.go @@ -0,0 +1,198 @@ +package graceful + +import ( + "io" + "net" + "sync" + "time" +) + +type listener struct { + net.Listener +} + +type gracefulConn interface { + gracefulShutdown() +} + +// Wrap an arbitrary net.Listener for use with graceful shutdowns. All +// net.Conn's Accept()ed by this listener will be auto-wrapped as if WrapConn() +// were called on them. +func WrapListener(l net.Listener) net.Listener { + return listener{l} +} + +func (l listener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + + return WrapConn(conn), nil +} + +/* +Wrap an arbitrary connection for use with graceful shutdowns. The graceful +shutdown process will ensure that this connection is closed before terminating +the process. + +In order to use this function, you must call SetReadDeadline() before the call +to Read() you might make to read a new request off the wire. The connection is +eligible for abrupt closing at any point between when the call to +SetReadDeadline() returns and when the call to Read returns with new data. It +does not matter what deadline is given to SetReadDeadline()--the default HTTP +server provided by this package sets a deadline far into the future when a +deadline is not provided, for instance. + +Unfortunately, this means that it's difficult to use SetReadDeadline() in a +great many perfectly reasonable circumstances, such as to extend a deadline +after more data has been read, without the connection being eligible for +"graceful" termination at an undesirable time. Since this package was written +explicitly to target net/http, which does not as of this writing do any of this, +fixing the semantics here does not seem especially urgent. + +As an optimization for net/http over TCP, if the input connection supports the +ReadFrom() function, the returned connection will as well. This allows the net +package to use sendfile(2) on certain platforms in certain circumstances. +*/ +func WrapConn(c net.Conn) net.Conn { + wg.Add(1) + + nc := conn{ + Conn: c, + closing: make(chan struct{}), + } + + if _, ok := c.(io.ReaderFrom); ok { + c = &sendfile{nc} + } else { + c = &nc + } + + go c.(gracefulConn).gracefulShutdown() + + return c +} + +type connstate int + +/* +State diagram. (Waiting) is the starting state. + +(Waiting) -----Read()-----> Working ---+ + | ^ / | ^ Read() + | \ / | +----+ + kill SetReadDeadline() kill + | | +-----+ + V V V Read() + Dead <-SetReadDeadline()-- Dying ----+ + ^ + | + +--Close()--- [from any state] + +*/ + +const ( + // Waiting for more data, and eligible for killing + csWaiting connstate = iota + // In the middle of a connection + csWorking + // Kill has been requested, but waiting on request to finish up + csDying + // Connection is gone forever. Also used when a connection gets hijacked + csDead +) + +type conn struct { + net.Conn + m sync.Mutex + state connstate + closing chan struct{} +} +type sendfile struct{ conn } + +func (c *conn) gracefulShutdown() { + select { + case <-kill: + case <-c.closing: + return + } + c.m.Lock() + defer c.m.Unlock() + + switch c.state { + case csWaiting: + c.unlockedClose(true) + case csWorking: + c.state = csDying + } +} + +func (c *conn) unlockedClose(closeConn bool) { + if closeConn { + c.Conn.Close() + } + close(c.closing) + wg.Done() + c.state = csDead +} + +// We do some hijinks to support hijacking. The semantics here is that any +// connection that gets hijacked is dead to us: we return the raw net.Conn and +// stop tracking the connection entirely. +type hijackConn interface { + hijack() net.Conn +} + +func (c *conn) hijack() net.Conn { + c.m.Lock() + defer c.m.Unlock() + if c.state != csDead { + close(c.closing) + wg.Done() + c.state = csDead + } + return c.Conn +} + +func (c *conn) Read(b []byte) (n int, err error) { + defer func() { + c.m.Lock() + defer c.m.Unlock() + + if c.state == csWaiting { + c.state = csWorking + } + }() + + return c.Conn.Read(b) +} +func (c *conn) Close() error { + defer func() { + c.m.Lock() + defer c.m.Unlock() + + if c.state != csDead { + c.unlockedClose(false) + } + }() + return c.Conn.Close() +} +func (c *conn) SetReadDeadline(t time.Time) error { + defer func() { + c.m.Lock() + defer c.m.Unlock() + switch c.state { + case csDying: + c.unlockedClose(false) + case csWorking: + c.state = csWaiting + } + }() + return c.Conn.SetReadDeadline(t) +} + +func (s *sendfile) ReadFrom(r io.Reader) (int64, error) { + // conn.Conn.KHAAAAAAAANNNNNN + return s.conn.Conn.(io.ReaderFrom).ReadFrom(r) +} diff --git a/graceful/net_test.go b/graceful/net_test.go new file mode 100644 index 0000000..d6e7208 --- /dev/null +++ b/graceful/net_test.go @@ -0,0 +1,198 @@ +package graceful + +import ( + "io" + "net" + "strings" + "testing" + "time" +) + +var b = make([]byte, 0) + +func connify(c net.Conn) *conn { + switch c.(type) { + case (*conn): + return c.(*conn) + case (*sendfile): + return &c.(*sendfile).conn + default: + panic("IDK") + } +} + +func assertState(t *testing.T, n net.Conn, st connstate) { + c := connify(n) + c.m.Lock() + defer c.m.Unlock() + if c.state != st { + t.Fatalf("conn was %v, but expected %v", c.state, st) + } +} + +// Not super happy about making the tests dependent on the passing of time, but +// I'm not really sure what else to do. + +func expectCall(t *testing.T, ch <-chan struct{}, name string) { + select { + case <-ch: + case <-time.After(5 * time.Millisecond): + t.Fatalf("Expected call to %s", name) + } +} + +func TestCounting(t *testing.T) { + kill = make(chan struct{}) + c := WrapConn(fakeConn{}) + ch := make(chan struct{}) + + go func() { + wg.Wait() + ch <- struct{}{} + }() + + select { + case <-ch: + t.Fatal("Expected connection to keep us from quitting") + case <-time.After(5 * time.Millisecond): + } + + c.Close() + expectCall(t, ch, "wg.Wait()") +} + +func TestStateTransitions1(t *testing.T) { + kill = make(chan struct{}) + ch := make(chan struct{}) + + onclose := make(chan struct{}) + read := make(chan struct{}) + deadline := make(chan struct{}) + c := WrapConn(fakeConn{ + onClose: func() { + onclose <- struct{}{} + }, + onRead: func() { + read <- struct{}{} + }, + onSetReadDeadline: func() { + deadline <- struct{}{} + }, + }) + + go func() { + wg.Wait() + ch <- struct{}{} + }() + + assertState(t, c, csWaiting) + + // Waiting + Read() = Working + go c.Read(b) + expectCall(t, read, "c.Read()") + assertState(t, c, csWorking) + + // Working + SetReadDeadline() = Waiting + go c.SetReadDeadline(time.Now()) + expectCall(t, deadline, "c.SetReadDeadline()") + assertState(t, c, csWaiting) + + // Waiting + kill = Dead + close(kill) + expectCall(t, onclose, "c.Close()") + assertState(t, c, csDead) + + expectCall(t, ch, "wg.Wait()") +} + +func TestStateTransitions2(t *testing.T) { + kill = make(chan struct{}) + ch := make(chan struct{}) + onclose := make(chan struct{}) + read := make(chan struct{}) + deadline := make(chan struct{}) + c := WrapConn(fakeConn{ + onClose: func() { + onclose <- struct{}{} + }, + onRead: func() { + read <- struct{}{} + }, + onSetReadDeadline: func() { + deadline <- struct{}{} + }, + }) + + go func() { + wg.Wait() + ch <- struct{}{} + }() + + assertState(t, c, csWaiting) + + // Waiting + Read() = Working + go c.Read(b) + expectCall(t, read, "c.Read()") + assertState(t, c, csWorking) + + // Working + Read() = Working + go c.Read(b) + expectCall(t, read, "c.Read()") + assertState(t, c, csWorking) + + // Working + kill = Dying + close(kill) + time.Sleep(5 * time.Millisecond) + assertState(t, c, csDying) + + // Dying + Read() = Dying + go c.Read(b) + expectCall(t, read, "c.Read()") + assertState(t, c, csDying) + + // Dying + SetReadDeadline() = Dead + go c.SetReadDeadline(time.Now()) + expectCall(t, deadline, "c.SetReadDeadline()") + assertState(t, c, csDead) + + expectCall(t, ch, "wg.Wait()") +} + +func TestHijack(t *testing.T) { + kill = make(chan struct{}) + fake := fakeConn{} + c := WrapConn(fake) + ch := make(chan struct{}) + + go func() { + wg.Wait() + ch <- struct{}{} + }() + + cc := connify(c) + if _, ok := cc.hijack().(fakeConn); !ok { + t.Error("Expected original connection back out") + } + assertState(t, c, csDead) + expectCall(t, ch, "wg.Wait()") +} + +type fakeSendfile struct { + fakeConn +} + +func (f fakeSendfile) ReadFrom(r io.Reader) (int64, error) { + return 0, nil +} + +func TestReadFrom(t *testing.T) { + kill = make(chan struct{}) + c := WrapConn(fakeSendfile{}) + r := strings.NewReader("Hello world") + + if rf, ok := c.(io.ReaderFrom); ok { + rf.ReadFrom(r) + } else { + t.Fatal("Expected a ReaderFrom in return") + } +} diff --git a/graceful/signal.go b/graceful/signal.go new file mode 100644 index 0000000..9a764a5 --- /dev/null +++ b/graceful/signal.go @@ -0,0 +1,117 @@ +package graceful + +import ( + "log" + "os" + "os/signal" + "sync" +) + +// This is the channel that the connections select on. When it is closed, the +// connections should gracefully exit. +var kill = make(chan struct{}) + +// This is the channel that the Wait() function selects on. It should only be +// closed once all the posthooks have been called. +var wait = make(chan struct{}) + +// This is the WaitGroup that indicates when all the connections have gracefully +// shut down. +var wg sync.WaitGroup + +// This lock protects the list of pre- and post- hooks below. +var hookLock sync.Mutex +var prehooks = make([]func(), 0) +var posthooks = make([]func(), 0) + +var sigchan = make(chan os.Signal, 1) + +func init() { + AddSignal(os.Interrupt) + go waitForSignal() +} + +// Add the given signal to the set of signals that trigger a graceful shutdown. +// Note that for convenience the default interrupt (SIGINT) handler is installed +// at package load time, and unless you call ResetSignals() will be listened for +// in addition to any signals you provide by calling this function. +func AddSignal(sig ...os.Signal) { + signal.Notify(sigchan, sig...) +} + +// Reset the list of signals that trigger a graceful shutdown. Useful if, for +// instance, you don't want to use the default interrupt (SIGINT) handler. Since +// we necessarily install the SIGINT handler before you have a chance to call +// ResetSignals(), there will be a brief window during which the set of signals +// this package listens for will not be as you intend. Therefore, if you intend +// on using this function, we encourage you to call it as soon as possible. +func ResetSignals() { + signal.Stop(sigchan) +} + +type userShutdown struct{} + +func (u userShutdown) String() string { + return "application initiated shutdown" +} +func (u userShutdown) Signal() {} + +// Manually trigger a shutdown from your application. Like Wait(), blocks until +// all connections have gracefully shut down. +func Shutdown() { + sigchan <- userShutdown{} + <-wait +} + +// Register a function to be called before any of this package's normal shutdown +// actions. All listeners will be called in the order they were added, from a +// single goroutine. +func PreHook(f func()) { + hookLock.Lock() + defer hookLock.Unlock() + + prehooks = append(prehooks, f) +} + +// Register a function to be called after all of this package's normal shutdown +// actions. All listeners will be called in the order they were added, from a +// single goroutine, and are guaranteed to be called after all listening +// connections have been closed, but before Wait() returns. +// +// If you've Hijack()ed any connections that must be gracefully shut down in +// some other way (since this library disowns all hijacked connections), it's +// reasonable to use a PostHook() to signal and wait for them. +func PostHook(f func()) { + hookLock.Lock() + defer hookLock.Unlock() + + posthooks = append(posthooks, f) +} + +func waitForSignal() { + sig := <-sigchan + log.Printf("Received %v, gracefully shutting down!", sig) + + hookLock.Lock() + defer hookLock.Unlock() + + for _, f := range prehooks { + f() + } + + close(kill) + wg.Wait() + + for _, f := range posthooks { + f() + } + + close(wait) +} + +// Wait for all connections to gracefully shut down. This is commonly called at +// the bottom of the main() function to prevent the program from exiting +// prematurely. +func Wait() { + <-wait +} From f8244ffe6611d5c2a5a3e768ece09ca8fc17dbbf Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 15 Mar 2014 13:23:05 -0700 Subject: [PATCH 005/217] Reference graceful shutdown golang issue Apparently stdlib support will be in Go 1.3. Yay! --- graceful/graceful.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graceful/graceful.go b/graceful/graceful.go index 121e307..0d51726 100644 --- a/graceful/graceful.go +++ b/graceful/graceful.go @@ -11,7 +11,7 @@ by carefully reading the sequence of function calls net/http happened to use as of this writing and finding enough surface area with which to add appropriate behavior. There's a very good chance that this package will cease to work in future versions of Go, but with any luck the standard library will add support -of its own by then. +of its own by then (https://code.google.com/p/go/issues/detail?id=4674). If you're interested in figuring out how this package works, we suggest you read the documentation for WrapConn() and net.go. From ed438ca532172ee07a9ffccba9328e7baf858b5f Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 17 Mar 2014 00:00:23 -0700 Subject: [PATCH 006/217] A wild web framework appears They say that every programmer builds a web framework at some point. This one is mine. The basic idea behind this one is that I wanted a Sinatra for Go, and I couldn't find one anywhere. Furthermore, net/http is in many ways really close to what I want out of a Sinatra-in-Go, and many of the frameworks I did find seemed to reinvent too much, or were incompatible with net/http in weird ways, or used too much questionable reflection magic. So long story short, I wrote my own. This implementation is only half-baked, and among other things it's missing a whole lot of tests. --- web/middleware.go | 184 ++++++++++++++++++++++++++++++ web/mux.go | 78 +++++++++++++ web/pattern.go | 121 ++++++++++++++++++++ web/router.go | 277 ++++++++++++++++++++++++++++++++++++++++++++++ web/web.go | 122 ++++++++++++++++++++ 5 files changed, 782 insertions(+) create mode 100644 web/middleware.go create mode 100644 web/mux.go create mode 100644 web/pattern.go create mode 100644 web/router.go create mode 100644 web/web.go diff --git a/web/middleware.go b/web/middleware.go new file mode 100644 index 0000000..fe8da8c --- /dev/null +++ b/web/middleware.go @@ -0,0 +1,184 @@ +package web + +import ( + "fmt" + "log" + "net/http" + "sync" +) + +// Maximum size of the pool of spare middleware stacks +const mPoolSize = 32 + +type mLayer struct { + fn func(*C, http.Handler) http.Handler + name string +} + +type mStack struct { + lock sync.Mutex + stack []mLayer + pool chan *cStack + router Handler +} + +// Constructing a middleware stack involves a lot of allocations: at the very +// least each layer will have to close over the layer after (inside) it, and +// perhaps a context object. Instead of doing this on every request, let's cache +// fully assembled middleware stacks (the "c" stands for "cached"). +type cStack struct { + C + m http.Handler +} + +func (s *cStack) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.C = C{} + s.m.ServeHTTP(w, r) +} +func (s *cStack) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { + s.C = c + s.m.ServeHTTP(w, r) +} + +func (m *mStack) appendLayer(name string, fn interface{}) { + var ml mLayer + ml.name = name + switch fn.(type) { + case func(http.Handler) http.Handler: + unwrapped := fn.(func(http.Handler) http.Handler) + ml.fn = func(c *C, h http.Handler) http.Handler { + return unwrapped(h) + } + case func(*C, http.Handler) http.Handler: + ml.fn = fn.(func(*C, http.Handler) http.Handler) + default: + log.Fatalf(`Unknown middleware type %v. Expected a function `+ + `with signature "func(http.Handler) http.Handler" or `+ + `"func(*web.C, http.Handler) http.Handler".`, fn) + } + m.stack = append(m.stack, ml) +} + +func (m *mStack) findLayer(name string) int { + for i, middleware := range m.stack { + if middleware.name == name { + return i + } + } + return -1 +} + +func (m *mStack) invalidate() { + old := m.pool + m.pool = make(chan *cStack, mPoolSize) + close(old) + for _ = range old { + } +} + +func (m *mStack) newStack() *cStack { + m.lock.Lock() + defer m.lock.Unlock() + + cs := cStack{} + router := m.router + + cs.m = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + router.ServeHTTPC(cs.C, w, r) + }) + for i := len(m.stack) - 1; i >= 0; i-- { + cs.m = m.stack[i].fn(&cs.C, cs.m) + } + + return &cs +} + +func (m *mStack) alloc() *cStack { + select { + case cs := <-m.pool: + if cs == nil { + return m.alloc() + } + return cs + default: + return m.newStack() + } +} + +func (m *mStack) release(cs *cStack) { + // It's possible that the pool has been invalidated and therefore + // closed, in which case we'll start panicing, which is dumb. I'm not + // sure this is actually better than just grabbing a lock, but whatever. + defer func() { + recover() + }() + select { + case m.pool <- cs: + default: + } +} + +// Append the given middleware to the middleware stack. See the documentation +// for type Mux for a list of valid middleware types. +// +// No attempt is made to enforce the uniqueness of middleware names. +func (m *mStack) Use(name string, middleware interface{}) { + m.lock.Lock() + defer m.lock.Unlock() + m.appendLayer(name, middleware) + m.invalidate() +} + +// Insert the given middleware immediately before a given existing middleware in +// the stack. See the documentation for type Mux for a list of valid middleware +// types. Returns an error if no middleware has the name given by "before." +// +// No attempt is made to enforce the uniqueness of names. If the insertion point +// is ambiguous, the first (outermost) one is chosen. +func (m *mStack) Insert(name string, middleware interface{}, before string) error { + m.lock.Lock() + defer m.lock.Unlock() + i := m.findLayer(before) + if i < 0 { + return fmt.Errorf("web: unknown middleware %v", before) + } + + m.appendLayer(name, middleware) + inserted := m.stack[len(m.stack)-1] + copy(m.stack[i+1:], m.stack[i:]) + m.stack[i] = inserted + + m.invalidate() + return nil +} + +// Remove the given named middleware from the middleware stack. Returns an error +// if there is no middleware by that name. +// +// If the name of the middleware to delete is ambiguous, the first (outermost) +// one is chosen. +func (m *mStack) Abandon(name string) error { + m.lock.Lock() + defer m.lock.Unlock() + i := m.findLayer(name) + if i < 0 { + return fmt.Errorf("web: unknown middleware %v", name) + } + + copy(m.stack[i:], m.stack[i+1:]) + m.stack = m.stack[:len(m.stack)-1 : len(m.stack)] + + m.invalidate() + return nil +} + +// Returns a list of middleware currently in use. +func (m *mStack) Middleware() []string { + m.lock.Lock() + defer m.lock.Unlock() + stack := make([]string, len(m.stack)) + for i, ml := range m.stack { + stack[i] = ml.name + } + return stack +} diff --git a/web/mux.go b/web/mux.go new file mode 100644 index 0000000..d95e929 --- /dev/null +++ b/web/mux.go @@ -0,0 +1,78 @@ +package web + +import ( + "net/http" +) + +/* +An HTTP multiplexer, much like net/http's ServeMux. + +Routes may be added using any of the various HTTP-method-specific functions. +When processing a request, when iterating in insertion order the first route +that matches both the request's path and method is used. + +There are two other differences worth mentioning between web.Mux and +http.ServeMux. First, string patterns (i.e., Sinatra-like patterns) must match +exactly: the "rooted subtree" behavior of ServeMux is not implemented. Secondly, +unlike ServeMux, Mux does not support Host-specific patterns. + +If you require any of these features, remember that you are free to mix and +match muxes at any part of the stack. + +In order to provide a sane API, many functions on Mux take interface{}'s. This +is obviously not a very satisfying solution, but it's probably the best we can +do for now. Instead of duplicating documentation on each method, the types +accepted by those functions are documented here. + +A middleware (the untyped parameter in Use() and Insert()) must be one of the +following types: + - func(http.Handler) http.Handler + - func(c *web.C, http.Handler) http.Handler +All of the route-adding functions on Mux take two untyped parameters: pattern +and handler. Pattern must be one of the following types: + - string (interpreted as a Sinatra pattern) + - regexp.Regexp + - web.Pattern +Handler must be one of the following types: + - http.Handler + - web.Handler + - func(w http.ResponseWriter, r *http.Request) + - func(c web.C, w http.ResponseWriter, r *http.Request) +*/ +type Mux struct { + mStack + router +} + +// Create a new Mux without any routes or middleware. +func New() *Mux { + mux := Mux{ + mStack: mStack{ + stack: make([]mLayer, 0), + pool: make(chan *cStack), + }, + router: router{ + routes: make([]route, 0), + notFound: parseHandler(http.NotFound), + }, + } + mux.mStack.router = HandlerFunc(mux.router.route) + return &mux +} + +// Serve a request with the given Mux. Satisfies the http.Handler interface. +func (m *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) { + stack := m.mStack.alloc() + defer m.mStack.release(stack) + + stack.ServeHTTP(w, r) +} + +// Serve a context dependent request with the given Mux. Satisfies the +// web.Handler interface. +func (m *Mux) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { + stack := m.mStack.alloc() + defer m.mStack.release(stack) + + stack.ServeHTTPC(c, w, r) +} diff --git a/web/pattern.go b/web/pattern.go new file mode 100644 index 0000000..3733765 --- /dev/null +++ b/web/pattern.go @@ -0,0 +1,121 @@ +package web + +import ( + "fmt" + "net/http" + "regexp" + "strings" +) + +type regexpPattern struct { + re *regexp.Regexp + names []string +} + +func (p regexpPattern) Prefix() string { + prefix, _ := p.re.LiteralPrefix() + return prefix +} +func (p regexpPattern) Match(r *http.Request, c *C) bool { + matches := p.re.FindStringSubmatch(r.URL.Path) + if matches == nil { + return false + } + + if c.UrlParams == nil && len(matches) > 0 { + c.UrlParams = make(map[string]string, len(matches)-1) + } + for i := 1; i < len(matches); i++ { + c.UrlParams[p.names[i]] = matches[i] + } + return true +} + +func parseRegexpPattern(re *regexp.Regexp) regexpPattern { + rnames := re.SubexpNames() + // We have to make our own copy since package regexp forbids us + // from scribbling over the slice returned by SubexpNames(). + names := make([]string, len(rnames)) + for i, rname := range rnames { + if rname == "" { + rname = fmt.Sprintf("$%d", i) + } + names[i] = rname + } + return regexpPattern{ + re: re, + names: names, + } +} + +type stringPattern struct { + raw string + pats []string + literals []string + isPrefix bool +} + +func (s stringPattern) Prefix() string { + return s.literals[0] +} + +func (s stringPattern) Match(r *http.Request, c *C) bool { + path := r.URL.Path + matches := make([]string, len(s.pats)) + for i := 0; i < len(s.pats); i++ { + if !strings.HasPrefix(path, s.literals[i]) { + return false + } + path = path[len(s.literals[i]):] + + m := strings.IndexRune(path, '/') + if m == -1 { + m = len(path) + } + if m == 0 { + // Empty strings are not matches, otherwise routes like + // "/:foo" would match the path "/" + return false + } + matches[i] = path[:m] + path = path[m:] + } + // There's exactly one more literal than pat. + if s.isPrefix { + if strings.HasPrefix(path, s.literals[len(s.pats)]) { + return false + } + } else { + if path != s.literals[len(s.pats)] { + return false + } + } + + if c.UrlParams == nil && len(matches) > 0 { + c.UrlParams = make(map[string]string, len(matches)-1) + } + for i, match := range matches { + c.UrlParams[s.pats[i]] = match + } + return true +} + +func parseStringPattern(s string, isPrefix bool) stringPattern { + matches := patternRe.FindAllStringSubmatchIndex(s, -1) + pats := make([]string, len(matches)) + literals := make([]string, len(matches)+1) + n := 0 + for i, match := range matches { + a, b := match[2], match[3] + literals[i] = s[n : a-1] // Need to leave off the colon + pats[i] = s[a:b] + n = b + } + literals[len(matches)] = s[n:] + return stringPattern{ + raw: s, + pats: pats, + literals: literals, + isPrefix: isPrefix, + } +} diff --git a/web/router.go b/web/router.go new file mode 100644 index 0000000..0db15c4 --- /dev/null +++ b/web/router.go @@ -0,0 +1,277 @@ +package web + +import ( + "log" + "net/http" + "regexp" + "strings" + "sync" +) + +type method int + +const ( + mCONNECT method = 1 << iota + mDELETE + mGET + mHEAD + mOPTIONS + mPATCH + mPOST + mPUT + mTRACE + // We only natively support the methods above, but we pass through other + // methods. This constant pretty much only exists for the sake of mALL. + mIDK + + mALL method = mCONNECT | mDELETE | mGET | mHEAD | mOPTIONS | mPATCH | + mPOST | mPUT | mTRACE | mIDK +) + +type route struct { + // Theory: most real world routes have a string prefix which is both + // cheap(-ish) to test against and pretty selective. And, conveniently, + // both regexes and string patterns give us this out-of-box. + prefix string + method method + pattern Pattern + handler Handler +} + +type router struct { + lock sync.Mutex + routes []route + notFound Handler +} + +// A pattern determines whether or not a given request matches some criteria. +// They are often used in routes, which are essentially (pattern, methodSet, +// handler) tuples. If the method and pattern match, the given handler is used. +// +// Built-in implementations of this interface are used to implement regular +// expression and string matching. +type Pattern interface { + // In practice, most real-world routes have a string prefix that can be + // used to quickly determine if a pattern is an eligible match. The + // router uses the result of this function to optimize away calls to the + // full Match function, which is likely much more expensive to compute. + // If your Pattern does not support prefixes, this function should + // return the empty string. + Prefix() string + // Returns true if the request satisfies the pattern. This function is + // free to examine both the request and the context to make this + // decision. After it is certain that the request matches, this function + // should mutate or create c.UrlParams if necessary. + Match(r *http.Request, c *C) bool +} + +var patternRe = regexp.MustCompile(`/:([^/]+)`) + +func parsePattern(p interface{}, isPrefix bool) Pattern { + switch p.(type) { + case Pattern: + return p.(Pattern) + case *regexp.Regexp: + return parseRegexpPattern(p.(*regexp.Regexp)) + case string: + return parseStringPattern(p.(string), isPrefix) + default: + log.Fatalf("Unknown pattern type %v. Expected a web.Pattern, "+ + "regexp.Regexp, or a string.", p) + } + panic("log.Fatalf does not return") +} + +type netHttpWrap struct { + http.Handler +} + +func (h netHttpWrap) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.Handler.ServeHTTP(w, r) +} +func (h netHttpWrap) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { + h.Handler.ServeHTTP(w, r) +} + +func parseHandler(h interface{}) Handler { + switch h.(type) { + case Handler: + return h.(Handler) + case http.Handler: + return netHttpWrap{h.(http.Handler)} + case func(c C, w http.ResponseWriter, r *http.Request): + f := h.(func(c C, w http.ResponseWriter, r *http.Request)) + return HandlerFunc(f) + case func(w http.ResponseWriter, r *http.Request): + f := h.(func(w http.ResponseWriter, r *http.Request)) + return netHttpWrap{http.HandlerFunc(f)} + default: + log.Fatalf("Unknown handler type %v. Expected a web.Handler, "+ + "a http.Handler, or a function with signature func(C, "+ + "http.ResponseWriter, *http.Request) or "+ + "func(http.ResponseWriter, http.Request)", h) + } + panic("log.Fatalf does not return") +} + +func httpMethod(mname string) method { + switch mname { + case "CONNECT": + return mCONNECT + case "DELETE": + return mDELETE + case "GET": + return mGET + case "HEAD": + return mHEAD + case "OPTIONS": + return mOPTIONS + case "PATCH": + return mPATCH + case "POST": + return mPOST + case "PUT": + return mPUT + case "TRACE": + return mTRACE + default: + return mIDK + } +} + +func (rt *router) route(c C, w http.ResponseWriter, r *http.Request) { + m := httpMethod(r.Method) + for _, route := range rt.routes { + if route.method&m == 0 || + !strings.HasPrefix(r.URL.Path, route.prefix) || + !route.pattern.Match(r, &c) { + continue + } + route.handler.ServeHTTPC(c, w, r) + return + } + + rt.notFound.ServeHTTPC(c, w, r) +} + +func (rt *router) handleUntyped(p interface{}, m method, h interface{}) { + pat := parsePattern(p, false) + rt.handle(pat, m, parseHandler(h)) +} + +func (rt *router) handle(p Pattern, m method, h Handler) { + // We're being a little sloppy here: we assume that pointer assignments + // are atomic, and that there is no way a locked append here can affect + // another goroutine which looked at rt.routes without a lock. + rt.lock.Lock() + defer rt.lock.Unlock() + rt.routes = append(rt.routes, route{ + prefix: p.Prefix(), + method: m, + pattern: p, + handler: h, + }) +} + +// This is a bit silly, but I've renamed the method receivers in the public +// functions here "m" instead of the standard "rt", since they will eventually +// be shown on the documentation for the Mux that they are included in. + +// Dispatch to the given handler when the pattern matches, regardless of HTTP +// method. See the documentation for type Mux for a description of what types +// are accepted for pattern and handler. +func (m *router) Handle(pattern interface{}, handler interface{}) { + m.handleUntyped(pattern, mALL, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// CONNECT. See the documentation for type Mux for a description of what types +// are accepted for pattern and handler. +func (m *router) Connect(pattern interface{}, handler interface{}) { + m.handleUntyped(pattern, mCONNECT, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// DELETE. See the documentation for type Mux for a description of what types +// are accepted for pattern and handler. +func (m *router) Delete(pattern interface{}, handler interface{}) { + m.handleUntyped(pattern, mDELETE, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// GET. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +func (m *router) Get(pattern interface{}, handler interface{}) { + m.handleUntyped(pattern, mGET, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// HEAD. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +func (m *router) Head(pattern interface{}, handler interface{}) { + m.handleUntyped(pattern, mHEAD, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// OPTIONS. See the documentation for type Mux for a description of what types +// are accepted for pattern and handler. +func (m *router) Options(pattern interface{}, handler interface{}) { + m.handleUntyped(pattern, mOPTIONS, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// PATCH. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +func (m *router) Patch(pattern interface{}, handler interface{}) { + m.handleUntyped(pattern, mPATCH, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// POST. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +func (m *router) Post(pattern interface{}, handler interface{}) { + m.handleUntyped(pattern, mPOST, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// PUT. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +func (m *router) Put(pattern interface{}, handler interface{}) { + m.handleUntyped(pattern, mPUT, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// TRACE. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +func (m *router) Trace(pattern interface{}, handler interface{}) { + m.handleUntyped(pattern, mTRACE, handler) +} + +/* +Dispatch to the given handler when the given (Sinatra-like) pattern matches a +prefix of the path. This function explicitly takes a string parameter since you +can implement this behavior with a regular expression using the standard +Handle() function. + +This function is probably most helpful when implementing sub-routing: an admin +application, for instance, can expose a single handler, and can be hooked up all +at once by attaching a sub-route at "/admin". + +Notably, this function does not strip the matched prefix from downstream +handlers, so in the above example the handler would recieve requests with path +"/admin/foo/bar", for example, instead of just "/foo/bar". Luckily, this is a +problem easily surmountable by middleware. + +See the documentation for type Mux for a description of what types are accepted +for handler. +*/ +func (m *router) Sub(pattern string, handler interface{}) { + pat := parsePattern(pattern, true) + m.handle(pat, mALL, parseHandler(handler)) +} + +// Set the fallback (i.e., 404) handler for this mux. See the documentation for +// type Mux for a description of what types are accepted for handler. +func (m *router) NotFound(handler interface{}) { + m.notFound = parseHandler(handler) +} diff --git a/web/web.go b/web/web.go new file mode 100644 index 0000000..60d17bb --- /dev/null +++ b/web/web.go @@ -0,0 +1,122 @@ +/* +Package web is a microframework inspired by Sinatra. + +The underlying philosophy behind this package is that net/http is a very good +HTTP library which is only missing a few features. If you disagree with this +statement (e.g., you think that the interfaces it exposes are not especially +good, or if you're looking for a comprehensive "batteries included" feature +list), you're likely not going to have a good time using this library. In that +spirit, we have attempted wherever possible to be compatible with net/http. You +should be able to insert any net/http compliant handler into this library, or +use this library with any other net/http compliant mux. + +This package attempts to solve three problems that net/http does not. First, it +allows you to specify URL patterns with Sinatra-like named wildcards and +regexps. Second, it allows you to write reconfigurable middleware stacks. And +finally, it allows you to attach additional context to requests, in a manner +that can be manipulated by both compliant middleware and handlers. + +A usage example: + + m := web.New() + +Use your favorite HTTP verbs: + + var legacyFooHttpHandler http.Handler // From elsewhere + m.Get("/foo", legacyFooHttpHandler) + m.Post("/bar", func(w http.ResponseWriter, r *http.Request) { + w.Write("Hello world!") + }) + +Bind parameters using either Sinatra-like patterns or regular expressions: + + m.Get("/hello/:name", func(c *web.C, w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello, %s!", c.UrlParams["name"]) + }) + pattern := regexp.MustCompile(`^/ip/(?P(?:\d{1,3}\.){3}\d{1,3})$`) + m.Get(pattern, func(c *web.C, w http.ResponseWriter, r *http.Request) { + fmt.Printf(w, "Info for IP address %s:", c.UrlParams["ip"]) + }) + +Middleware are functions that wrap http.Handlers, just like you'd use with raw +net/http. Middleware functions can optionally take a context parameter, which +will be threaded throughout the middleware stack and to the final handler, even +if not all of these things do not support contexts. Middleware are encouraged to +use the Env parameter to pass data to other middleware and to the final handler: + + m.Use(func(h http.Handler) http.Handler { + handler := func(w http.ResponseWriter, r *http.Request) { + log.Println("Before request") + h.ServeHTTP(w, r) + log.Println("After request") + } + return http.HandlerFunc(handler) + }) + m.Use(func(c *web.C, h http.Handler) http.Handler { + handler := func(w http.ResponseWriter, r *http.Request) { + cookie, err := r.Cookie("user") + if err == nil { + c.Env["user"] = cookie.Raw + } + h.ServeHTTP(w, r) + } + return http.HandlerFunc(handler) + }) + + m.Get("/baz", func(c web.C, w http.ResponseWriter, r *http.Request) { + if user, ok := c.Env["user"], ok { + w.Write("Hello " + string(user)) + } else { + w.Write("Hello Stranger!") + } + }) +*/ +package web + +import ( + "net/http" +) + +/* +Per-request context object. Threaded through all compliant middleware layers and +to the final request handler. + +As an implementation detail, references to these structs are reused between +requests to reduce allocation churn, but the maps they contain are created fresh +on every request. If you are closing over a context (especially relevant for +middleware), you should not close over either the UrlParams or Env objects, +instead accessing them through the context whenever they are required. +*/ +type C struct { + // The parameters parsed by the mux from the URL itself. In most cases, + // will contain a map from programmer-specified identifiers to the + // strings that matched those identifiers, but if a unnamed regex + // capture is used, it will be assigned to the special identifiers "$1", + // "$2", etc. + UrlParams map[string]string + // A free-form environment, similar to Rack or PEP 333's environments. + // Middleware layers are encouraged to pass data to downstream layers + // and other handlers using this map, and are even more strongly + // encouraged to document and maybe namespace they keys they use. + Env map[string]interface{} +} + +// A superset of net/http's http.Handler, which also includes a mechanism for +// serving requests with a context. If your handler does not support the use of +// contexts, we encourage you to use http.Handler instead. +type Handler interface { + http.Handler + ServeHTTPC(C, http.ResponseWriter, *http.Request) +} + +// Like net/http's http.HandlerFunc, but supports a context object. Implements +// both http.Handler and web.Handler free of charge. +type HandlerFunc func(C, http.ResponseWriter, *http.Request) + +func (h HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h(C{}, w, r) +} + +func (h HandlerFunc) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { + h(c, w, r) +} From b0feb9b8e2444806ab1f2cabfb74e68c49cf0420 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 17 Mar 2014 00:50:55 -0700 Subject: [PATCH 007/217] New and improved toplevel package The idea is to make goji super easy to use out of the box. --- README.md | 36 +++++++++--------- default.go | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++++ goji.go | 67 +++++++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 18 deletions(-) create mode 100644 default.go create mode 100644 goji.go diff --git a/README.md b/README.md index 7d6629d..6b42cd1 100644 --- a/README.md +++ b/README.md @@ -3,26 +3,26 @@ Goji Goji is a minimalistic web framework inspired by Sinatra. +Example +------- -Philosophy ----------- +```go +package main -Most of the design decisions in Goji can be traced back to the fundamental -philosopy that the Go standard library got things Mostly Right, and if it -didn't, it at least is good enough that it's not worth fighting. +import ( + "fmt" + "net/http" -Therefore, Goji leans heavily on the standard library, and in particular its -interfaces and idioms. You can expect to be able to use most of Goji in exactly -the manner you would use a comparable standard library function, and have it -function in exactly the way you would expect. + "github.com/zenazn/goji" + "github.com/zenazn/goji/web" +) -Also in this vein, Goji makes use of Go's `flag` package, and in particular the -default global flag set. Third party packages that have global state and squat -on global namespaces is something to be suspicious of, but the `flag` package is -also the closest thing Go has to a unified configuration API, and when used -tastefully it can make everyone's lives a bit easier. Wherever possible, the use -of these flags is opt-out, at the cost of additional complexity for the user. +func hello(c web.C, w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello, %s!", c.UrlParams["name"]) +} -Goji also makes an attempt to not be magical -- explicit is better than -implicit. Goji does make use of reflection and `interface{}`, but only when an -API would be impossible or cumbersome without it. +func main() { + goji.Get("/hello/:name", hello) + goji.Serve() +} +``` diff --git a/default.go b/default.go new file mode 100644 index 0000000..c43cccf --- /dev/null +++ b/default.go @@ -0,0 +1,108 @@ +package goji + +import ( + "github.com/zenazn/goji/web" +) + +// The default web.Mux. +var DefaultMux *web.Mux + +func init() { + DefaultMux = web.New() +} + +// Append the given middleware to the default Mux's middleware stack. See the +// documentation for web.Mux.Use for more informatino. +func Use(name string, middleware interface{}) { + DefaultMux.Use(name, middleware) +} + +// Insert the given middleware into the default Mux's middleware stack. See the +// documentation for web.Mux.Insert for more informatino. +func Insert(name string, middleware interface{}, before string) error { + return DefaultMux.Insert(name, middleware, before) +} + +// Remove the given middleware from the default Mux's middleware stack. See the +// documentation for web.Mux.Abandon for more informatino. +func Abandon(name string) error { + return DefaultMux.Abandon(name) +} + +// Retrieve the list of middleware from the default Mux's middleware stack. See +// the documentation for web.Mux.Middleware() for more information. +func Middleware() []string { + return DefaultMux.Middleware() +} + +// Add a route to the default Mux. See the documentation for web.Mux for more +// information about what types this function accepts. +func Handle(pattern interface{}, handler interface{}) { + DefaultMux.Handle(pattern, handler) +} + +// Add a CONNECT route to the default Mux. See the documentation for web.Mux for +// more information about what types this function accepts. +func Connect(pattern interface{}, handler interface{}) { + DefaultMux.Connect(pattern, handler) +} + +// Add a DELETE route to the default Mux. See the documentation for web.Mux for +// more information about what types this function accepts. +func Delete(pattern interface{}, handler interface{}) { + DefaultMux.Delete(pattern, handler) +} + +// Add a GET route to the default Mux. See the documentation for web.Mux for +// more information about what types this function accepts. +func Get(pattern interface{}, handler interface{}) { + DefaultMux.Get(pattern, handler) +} + +// Add a HEAD route to the default Mux. See the documentation for web.Mux for +// more information about what types this function accepts. +func Head(pattern interface{}, handler interface{}) { + DefaultMux.Head(pattern, handler) +} + +// Add a OPTIONS route to the default Mux. See the documentation for web.Mux for +// more information about what types this function accepts. +func Options(pattern interface{}, handler interface{}) { + DefaultMux.Options(pattern, handler) +} + +// Add a PATCH route to the default Mux. See the documentation for web.Mux for +// more information about what types this function accepts. +func Patch(pattern interface{}, handler interface{}) { + DefaultMux.Patch(pattern, handler) +} + +// Add a POST route to the default Mux. See the documentation for web.Mux for +// more information about what types this function accepts. +func Post(pattern interface{}, handler interface{}) { + DefaultMux.Post(pattern, handler) +} + +// Add a PUT route to the default Mux. See the documentation for web.Mux for +// more information about what types this function accepts. +func Put(pattern interface{}, handler interface{}) { + DefaultMux.Put(pattern, handler) +} + +// Add a TRACE route to the default Mux. See the documentation for web.Mux for +// more information about what types this function accepts. +func Trace(pattern interface{}, handler interface{}) { + DefaultMux.Trace(pattern, handler) +} + +// Add a sub-route to the default Mux. See the documentation for web.Mux.Sub +// for more information. +func Sub(pattern string, handler interface{}) { + DefaultMux.Sub(pattern, handler) +} + +// Set the NotFound handler for the default Mux. See the documentation for +// web.Mux.NotFound for more information. +func NotFound(handler interface{}) { + DefaultMux.NotFound(handler) +} diff --git a/goji.go b/goji.go new file mode 100644 index 0000000..e973f7c --- /dev/null +++ b/goji.go @@ -0,0 +1,67 @@ +/* +Package goji provides an out-of-box web server with reasonable defaults. + +Example: + package main + + import ( + "fmt" + "net/http" + + "github.com/zenazn/goji" + "github.com/zenazn/goji/web" + ) + + func hello(c web.C, w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello, %s!", c.UrlParams["name"]) + } + + func main() { + goji.Get("/hello/:name", hello) + goji.Serve() + } + +This package exists purely as a convenience to programmers who want to get +started as quickly as possible. It draws almost all of its code from goji's +subpackages, the most interesting of which is goji/web, and where most of the +documentation for the web framework lives. + +A side effect of this package's ease-of-use is the fact that it is opinionated. +If you don't like (or have outgrown) its opinions, it should be straightforward +to use the APIs of goji's subpackages to reimplement things to your liking. Both +methods of using this library are equally well supported. +*/ +package goji + +import ( + "flag" + "log" + "net/http" + + "github.com/zenazn/goji/bind" + "github.com/zenazn/goji/graceful" +) + +// Start Goji using reasonable defaults. +func Serve() { + if !flag.Parsed() { + flag.Parse() + } + + // Install our handler at the root of the standard net/http default mux. + // This allows packages like expvar to continue working as expected. + http.Handle("/", DefaultMux) + + listener := bind.Default() + log.Println("Starting Goji on", listener.Addr()) + + bind.Ready() + + err := graceful.Serve(listener, http.DefaultServeMux) + + if err != nil { + log.Fatal(err) + } + + graceful.Wait() +} From cbf1b4c8f91583fbaa11962bab83caaee632cdcc Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 17 Mar 2014 01:07:15 -0700 Subject: [PATCH 008/217] Features! Everyone loves features! --- README.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b42cd1..20b9d28 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ Goji ==== -Goji is a minimalistic web framework inspired by Sinatra. +Goji is a minimalistic web framework inspired by Sinatra. [Godoc][doc]. + +[doc]: http://godoc.org/github.com/zenazn/goji Example ------- @@ -26,3 +28,21 @@ func main() { goji.Serve() } ``` + + +Features +-------- + +* Compatible with `net/http` +* URL patterns (both Sinatra style `/foo/:bar` patterns and regular expressions) +* Reconfigurable middleware stack +* Context/environment objects threaded through middleware and handlers +* Automatic support for [Einhorn][einhorn], systemd, and [more][bind] +* [Graceful shutdown][graceful], and zero-downtime graceful reload when combined + with Einhorn. +* Ruby on Rails / jQuery style [parameter parsing][param] + +[einhorn]: https://github.com/stripe/einhorn +[bind]: http://godoc.org/github.com/zenazn/goji/bind +[graceful]: http://godoc.org/github.com/zenazn/goji/graceful +[param]: http://godoc.org/github.com/zenazn/goji/param From df704cd5faf0d0b396edc8d4d4544d0d68dc27ab Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 17 Mar 2014 01:41:02 -0700 Subject: [PATCH 009/217] TODOs So I don't forget. --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 20b9d28..b0df83e 100644 --- a/README.md +++ b/README.md @@ -46,3 +46,22 @@ Features [bind]: http://godoc.org/github.com/zenazn/goji/bind [graceful]: http://godoc.org/github.com/zenazn/goji/graceful [param]: http://godoc.org/github.com/zenazn/goji/param + + +Todo +---- + +Goji probably deserves a bit more love before anyone actually tries to use it. +Things that need doing include: + +* Support for omitting trailing slashes on routes which include them. +* Tests for `goji/web`. There are currently no tests. This probably means + `goji/web` is made of bugs. I'm sorry. +* Standard middleware implementations. I'm currently thinking: + * Request ID assigner: injects a request ID into the environment. + * Request logger: logs requests as they come in. Preferrably with request IDs + and maybe even with colors. + * Request timer: maybe part of the request logger. Time how long each request + takes and print it. Maybe with color. + * Error handler: recovers from panics, does something nice with the output. + * Healthcheck endpoint: Always returns OK. From 3fffa7df4ac1454eccd0a711599418c0cd3be6b6 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 22 Mar 2014 19:26:14 -0700 Subject: [PATCH 010/217] Middleware tests + bugfixes My tests caught some bugs! Amazing! --- web/middleware.go | 33 ++++-- web/middleware_test.go | 255 +++++++++++++++++++++++++++++++++++++++++ web/mux.go | 2 +- 3 files changed, 281 insertions(+), 9 deletions(-) create mode 100644 web/middleware_test.go diff --git a/web/middleware.go b/web/middleware.go index fe8da8c..b62c67a 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -28,7 +28,8 @@ type mStack struct { // fully assembled middleware stacks (the "c" stands for "cached"). type cStack struct { C - m http.Handler + m http.Handler + pool chan *cStack } func (s *cStack) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -72,6 +73,7 @@ func (m *mStack) invalidate() { old := m.pool m.pool = make(chan *cStack, mPoolSize) close(old) + // Bleed down the old pool so it gets GC'd for _ = range old { } } @@ -94,26 +96,41 @@ func (m *mStack) newStack() *cStack { } func (m *mStack) alloc() *cStack { + // This is a little sloppy: this is only safe if this pointer + // dereference is atomic. Maybe someday I'll replace it with + // sync/atomic, but for now I happen to know that on all the + // architecures I care about it happens to be atomic. + p := m.pool + var cs *cStack select { - case cs := <-m.pool: + case cs = <-p: + // This can happen if we race against an invalidation. It's + // completely peaceful, so long as we assume we can grab a cStack before + // our stack blows out. if cs == nil { return m.alloc() } - return cs default: - return m.newStack() + cs = m.newStack() } + + cs.pool = p + return cs } func (m *mStack) release(cs *cStack) { - // It's possible that the pool has been invalidated and therefore - // closed, in which case we'll start panicing, which is dumb. I'm not - // sure this is actually better than just grabbing a lock, but whatever. + if cs.pool != m.pool { + return + } + // It's possible that the pool has been invalidated (and closed) between + // the check above and now, in which case we'll start panicing, which is + // dumb. I'm not sure this is actually better than just grabbing a lock, + // but whatever. defer func() { recover() }() select { - case m.pool <- cs: + case cs.pool <- cs: default: } } diff --git a/web/middleware_test.go b/web/middleware_test.go new file mode 100644 index 0000000..747b541 --- /dev/null +++ b/web/middleware_test.go @@ -0,0 +1,255 @@ +package web + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + "time" +) + +func makeStack(ch chan string) *mStack { + router := func(c C, w http.ResponseWriter, r *http.Request) { + ch <- "router" + } + return &mStack{ + stack: make([]mLayer, 0), + pool: make(chan *cStack, mPoolSize), + router: HandlerFunc(router), + } +} + +func chanWare(ch chan string, s string) func(http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + ch <- s + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + } +} + +func simpleRequest(ch chan string, st *mStack) { + defer func() { + ch <- "end" + }() + r, _ := http.NewRequest("GET", "/", nil) + w := httptest.NewRecorder() + cs := st.alloc() + defer st.release(cs) + + cs.ServeHTTP(w, r) +} + +func assertOrder(t *testing.T, ch chan string, strings ...string) { + for i, s := range strings { + var v string + select { + case v = <-ch: + case <-time.After(5 * time.Millisecond): + t.Fatalf("Expected %q as %d'th value, but timed out", s, + i+1) + } + if s != v { + t.Errorf("%d'th value was %q, expected %q", i+1, v, s) + } + } +} + +func TestSimple(t *testing.T) { + t.Parallel() + + ch := make(chan string) + st := makeStack(ch) + st.Use("one", chanWare(ch, "one")) + st.Use("two", chanWare(ch, "two")) + go simpleRequest(ch, st) + assertOrder(t, ch, "one", "two", "router", "end") +} + +func TestTypes(t *testing.T) { + t.Parallel() + + ch := make(chan string) + st := makeStack(ch) + st.Use("one", func(h http.Handler) http.Handler { + return h + }) + st.Use("two", func(c *C, h http.Handler) http.Handler { + return h + }) +} + +func TestAddMore(t *testing.T) { + t.Parallel() + + ch := make(chan string) + st := makeStack(ch) + st.Use("one", chanWare(ch, "one")) + go simpleRequest(ch, st) + assertOrder(t, ch, "one", "router", "end") + + st.Use("two", chanWare(ch, "two")) + go simpleRequest(ch, st) + assertOrder(t, ch, "one", "two", "router", "end") + + st.Use("three", chanWare(ch, "three")) + st.Use("four", chanWare(ch, "four")) + go simpleRequest(ch, st) + assertOrder(t, ch, "one", "two", "three", "four", "router", "end") +} + +func TestInsert(t *testing.T) { + t.Parallel() + + ch := make(chan string) + st := makeStack(ch) + st.Use("one", chanWare(ch, "one")) + st.Use("two", chanWare(ch, "two")) + go simpleRequest(ch, st) + assertOrder(t, ch, "one", "two", "router", "end") + + err := st.Insert("sloth", chanWare(ch, "sloth"), "squirrel") + if err == nil { + t.Error("Expected error when referencing unknown middleware") + } + + st.Insert("middle", chanWare(ch, "middle"), "two") + st.Insert("start", chanWare(ch, "start"), "one") + go simpleRequest(ch, st) + assertOrder(t, ch, "start", "one", "middle", "two", "router", "end") +} + +func TestAbandon(t *testing.T) { + t.Parallel() + + ch := make(chan string) + st := makeStack(ch) + st.Use("one", chanWare(ch, "one")) + st.Use("two", chanWare(ch, "two")) + st.Use("three", chanWare(ch, "three")) + go simpleRequest(ch, st) + assertOrder(t, ch, "one", "two", "three", "router", "end") + + st.Abandon("two") + go simpleRequest(ch, st) + assertOrder(t, ch, "one", "three", "router", "end") + + err := st.Abandon("panda") + if err == nil { + t.Error("Expected error when deleting unknown middleware") + } + + st.Abandon("one") + st.Abandon("three") + go simpleRequest(ch, st) + assertOrder(t, ch, "router", "end") + + st.Use("one", chanWare(ch, "one")) + go simpleRequest(ch, st) + assertOrder(t, ch, "one", "router", "end") +} + +func TestMiddlewareList(t *testing.T) { + t.Parallel() + + ch := make(chan string) + st := makeStack(ch) + st.Use("one", chanWare(ch, "one")) + st.Use("two", chanWare(ch, "two")) + st.Insert("mid", chanWare(ch, "mid"), "two") + st.Insert("before", chanWare(ch, "before"), "mid") + st.Abandon("one") + + m := st.Middleware() + if !reflect.DeepEqual(m, []string{"before", "mid", "two"}) { + t.Error("Middleware list was not as expected") + } + + go simpleRequest(ch, st) + assertOrder(t, ch, "before", "mid", "two", "router", "end") +} + +// This is a pretty sketchtacular test +func TestCaching(t *testing.T) { + ch := make(chan string) + st := makeStack(ch) + cs1 := st.alloc() + cs2 := st.alloc() + if cs1 == cs2 { + t.Fatal("cs1 and cs2 are the same") + } + st.release(cs2) + cs3 := st.alloc() + if cs2 != cs3 { + t.Fatalf("Expected cs2 to equal cs3") + } + st.release(cs1) + st.release(cs3) + cs4 := st.alloc() + cs5 := st.alloc() + if cs4 != cs1 { + t.Fatal("Expected cs4 to equal cs1") + } + if cs5 != cs3 { + t.Fatal("Expected cs5 to equal cs3") + } +} + +func TestInvalidation(t *testing.T) { + ch := make(chan string) + st := makeStack(ch) + cs1 := st.alloc() + cs2 := st.alloc() + st.release(cs1) + st.invalidate() + cs3 := st.alloc() + if cs3 == cs1 { + t.Fatal("Expected cs3 to be fresh, instead got cs1") + } + st.release(cs2) + cs4 := st.alloc() + if cs4 == cs2 { + t.Fatal("Expected cs4 to be fresh, instead got cs2") + } +} + +func TestContext(t *testing.T) { + router := func(c C, w http.ResponseWriter, r *http.Request) { + if c.Env["reqId"].(int) != 2 { + t.Error("Request id was not 2 :(") + } + } + st := mStack{ + stack: make([]mLayer, 0), + pool: make(chan *cStack, mPoolSize), + router: HandlerFunc(router), + } + st.Use("one", func(c *C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if c.Env != nil || c.UrlParams != nil { + t.Error("Expected a clean context") + } + c.Env = make(map[string]interface{}) + c.Env["reqId"] = 1 + + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + }) + + st.Use("two", func(c *C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if c.Env == nil { + t.Error("Expected env from last middleware") + } + c.Env["reqId"] = c.Env["reqId"].(int) + 1 + + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + }) + ch := make(chan string) + go simpleRequest(ch, &st) + assertOrder(t, ch, "end") +} diff --git a/web/mux.go b/web/mux.go index d95e929..c8f8d2c 100644 --- a/web/mux.go +++ b/web/mux.go @@ -49,7 +49,7 @@ func New() *Mux { mux := Mux{ mStack: mStack{ stack: make([]mLayer, 0), - pool: make(chan *cStack), + pool: make(chan *cStack, mPoolSize), }, router: router{ routes: make([]route, 0), From e1ef76d6a80768110cb261b6afaff192fa731272 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 22 Mar 2014 19:30:18 -0700 Subject: [PATCH 011/217] Pattern tests + regexp hilarity Add tests for both string and regular expression patterns. Also, reimplement regexp.Regexp.Prefix() on top of the raw regexp/syntax representation, so we can get a little more information out of regexps: - Whether or not the regexp is left-anchored (at the beginning of the string) - What the prefix of the regular expression is, even for left-anchored expressions. We do this by running the regular expression instructions ourselves, more or less cargo-culting the original implementation from package regexp/syntax. Unfortunately it's ~impossible to make this abstraction non-leaky, because the regexp package doesn't give us information about whether or not it was constructed using POSIX or Perl syntax, for example, or if the longest-match setting was applied. The upshot is that regexps are now probably pretty performant-ish. Maybe. (I haven't actually benchmarked it). --- web/mux.go | 9 ++- web/pattern.go | 112 +++++++++++++++++++++++++++--- web/pattern_test.go | 163 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 274 insertions(+), 10 deletions(-) create mode 100644 web/pattern_test.go diff --git a/web/mux.go b/web/mux.go index c8f8d2c..00b5d68 100644 --- a/web/mux.go +++ b/web/mux.go @@ -31,7 +31,10 @@ following types: All of the route-adding functions on Mux take two untyped parameters: pattern and handler. Pattern must be one of the following types: - string (interpreted as a Sinatra pattern) - - regexp.Regexp + - regexp.Regexp. The library assumes that it is a Perl-style regexp that + is anchored on the left (i.e., the beginning of the string). If your + regexp is not anchored on the left, a hopefully-identical left-anchored + regexp will be created and used instead. - web.Pattern Handler must be one of the following types: - http.Handler @@ -44,6 +47,10 @@ type Mux struct { router } +// Sanity check types +var _ http.Handler = &Mux{} +var _ Handler = &Mux{} + // Create a new Mux without any routes or middleware. func New() *Mux { mux := Mux{ diff --git a/web/pattern.go b/web/pattern.go index 3733765..f6b11c7 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -1,28 +1,35 @@ package web import ( + "bytes" "fmt" + "log" "net/http" "regexp" + "regexp/syntax" "strings" ) type regexpPattern struct { - re *regexp.Regexp - names []string + re *regexp.Regexp + prefix string + names []string } func (p regexpPattern) Prefix() string { - prefix, _ := p.re.LiteralPrefix() - return prefix + return p.prefix } func (p regexpPattern) Match(r *http.Request, c *C) bool { matches := p.re.FindStringSubmatch(r.URL.Path) - if matches == nil { + if matches == nil || len(matches) == 0 { return false } - if c.UrlParams == nil && len(matches) > 0 { + if len(matches) == 1 { + return true + } + + if c.UrlParams == nil { c.UrlParams = make(map[string]string, len(matches)-1) } for i := 1; i < len(matches); i++ { @@ -31,7 +38,89 @@ func (p regexpPattern) Match(r *http.Request, c *C) bool { return true } +func (p regexpPattern) String() string { + return fmt.Sprintf("regexpPattern(%v)", p.re) +} + +/* +I'm sorry, dear reader. I really am. + +The problem here is to take an arbitrary regular expression and: +1. return a regular expression that is just like it, but left-anchored, + preferring to return the original if possible. +2. determine a string literal prefix that all matches of this regular expression + have, much like regexp.Regexp.Prefix(). Unfortunately, Prefix() does not work + in the presence of anchors, so we need to write it ourselves. + +What this actually means is that we need to sketch on the internals of the +standard regexp library to forcefully extract the information we want. + +Unfortunately, regexp.Regexp hides a lot of its state, so our abstraction is +going to be pretty leaky. The biggest leak is that we blindly assume that all +regular expressions are perl-style, not POSIX. This is probably Mostly True, and +I think most users of the library probably won't be able to notice. +*/ +func sketchOnRegex(re *regexp.Regexp) (*regexp.Regexp, string) { + rawRe := re.String() + sRe, err := syntax.Parse(rawRe, syntax.Perl) + if err != nil { + log.Printf("WARN(web): unable to parse regexp %v as perl. "+ + "This route might behave unexpectedly.", re) + return re, "" + } + sRe = sRe.Simplify() + p, err := syntax.Compile(sRe) + if err != nil { + log.Printf("WARN(web): unable to compile regexp %v. This "+ + "route might behave unexpectedly.", re) + return re, "" + } + if p.StartCond()&syntax.EmptyBeginText == 0 { + // I hope doing this is always legal... + newRe, err := regexp.Compile(`\A` + rawRe) + if err != nil { + log.Printf("WARN(web): unable to create a left-"+ + "anchored regexp from %v. This route might "+ + "behave unexpectedly", re) + return re, "" + } + re = newRe + } + + // Run the regular expression more or less by hand :( + pc := uint32(p.Start) + atStart := true + i := &p.Inst[pc] + var buf bytes.Buffer +Sadness: + for { + switch i.Op { + case syntax.InstEmptyWidth: + if !atStart { + break Sadness + } + case syntax.InstCapture, syntax.InstNop: + // nop! + case syntax.InstRune, syntax.InstRune1, syntax.InstRuneAny, + syntax.InstRuneAnyNotNL: + + atStart = false + if len(i.Rune) != 1 || + syntax.Flags(i.Arg)&syntax.FoldCase != 0 { + break Sadness + } + buf.WriteRune(i.Rune[0]) + default: + break Sadness + } + pc = i.Out + i = &p.Inst[pc] + } + return re, buf.String() +} + func parseRegexpPattern(re *regexp.Regexp) regexpPattern { + re, prefix := sketchOnRegex(re) rnames := re.SubexpNames() // We have to make our own copy since package regexp forbids us // from scribbling over the slice returned by SubexpNames(). @@ -43,8 +132,9 @@ func parseRegexpPattern(re *regexp.Regexp) regexpPattern { names[i] = rname } return regexpPattern{ - re: re, - names: names, + re: re, + prefix: prefix, + names: names, } } @@ -82,7 +172,7 @@ func (s stringPattern) Match(r *http.Request, c *C) bool { } // There's exactly one more literal than pat. if s.isPrefix { - if strings.HasPrefix(path, s.literals[len(s.pats)]) { + if !strings.HasPrefix(path, s.literals[len(s.pats)]) { return false } } else { @@ -100,6 +190,10 @@ func (s stringPattern) Match(r *http.Request, c *C) bool { return true } +func (s stringPattern) String() string { + return fmt.Sprintf("stringPattern(%q, %v)", s.raw, s.isPrefix) +} + func parseStringPattern(s string, isPrefix bool) stringPattern { matches := patternRe.FindAllStringSubmatchIndex(s, -1) pats := make([]string, len(matches)) diff --git a/web/pattern_test.go b/web/pattern_test.go new file mode 100644 index 0000000..7fbe132 --- /dev/null +++ b/web/pattern_test.go @@ -0,0 +1,163 @@ +package web + +import ( + "net/http" + "reflect" + "regexp" + "testing" +) + +func pt(url string, match bool, params map[string]string) patternTest { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + panic(err) + } + + return patternTest{ + r: req, + match: match, + c: &C{}, + cout: &C{UrlParams: params}, + } +} + +type patternTest struct { + r *http.Request + match bool + c *C + cout *C +} + +var patternTests = []struct { + pat Pattern + prefix string + tests []patternTest +}{ + // Regexp tests + {parseRegexpPattern(regexp.MustCompile("^/hello$")), + "/hello", []patternTest{ + pt("/hello", true, nil), + pt("/hell", false, nil), + pt("/hello/", false, nil), + pt("/hello/world", false, nil), + pt("/world", false, nil), + }}, + {parseRegexpPattern(regexp.MustCompile("^/hello/(?P[a-z]+)$")), + "/hello/", []patternTest{ + pt("/hello/world", true, map[string]string{ + "name": "world", + }), + pt("/hello/", false, nil), + pt("/hello/my/love", false, nil), + }}, + {parseRegexpPattern(regexp.MustCompile(`^/a(?P\d+)/b(?P\d+)/?$`)), + "/a", []patternTest{ + pt("/a1/b2", true, map[string]string{ + "a": "1", + "b": "2", + }), + pt("/a9001/b007/", true, map[string]string{ + "a": "9001", + "b": "007", + }), + pt("/a/b", false, nil), + pt("/a", false, nil), + pt("/squirrel", false, nil), + }}, + {parseRegexpPattern(regexp.MustCompile(`^/hello/([a-z]+)$`)), + "/hello/", []patternTest{ + pt("/hello/world", true, map[string]string{ + "$1": "world", + }), + pt("/hello/", false, nil), + }}, + {parseRegexpPattern(regexp.MustCompile("/hello")), + "/hello", []patternTest{ + pt("/hello", true, nil), + pt("/hell", false, nil), + pt("/hello/", true, nil), + pt("/hello/world", true, nil), + pt("/world/hello", false, nil), + }}, + + // String pattern tests + {parseStringPattern("/hello", false), + "/hello", []patternTest{ + pt("/hello", true, nil), + pt("/hell", false, nil), + pt("/hello/", false, nil), + pt("/hello/world", false, nil), + }}, + {parseStringPattern("/hello/:name", false), + "/hello/", []patternTest{ + pt("/hello/world", true, map[string]string{ + "name": "world", + }), + pt("/hell", false, nil), + pt("/hello/", false, nil), + pt("/hello/my/love", false, nil), + }}, + {parseStringPattern("/a/:a/b/:b", false), + "/a/", []patternTest{ + pt("/a/1/b/2", true, map[string]string{ + "a": "1", + "b": "2", + }), + pt("/a", false, nil), + pt("/a//b/", false, nil), + pt("/a/1/b/2/3", false, nil), + }}, + + // String sub-pattern tests + {parseStringPattern("/user/:user", true), + "/user/", []patternTest{ + pt("/user/bob", true, map[string]string{ + "user": "bob", + }), + pt("/user/bob/friends/123", true, map[string]string{ + "user": "bob", + }), + pt("/user/", false, nil), + pt("/user//", false, nil), + }}, + {parseStringPattern("/user/:user/friends", true), + "/user/", []patternTest{ + pt("/user/bob/friends", true, map[string]string{ + "user": "bob", + }), + pt("/user/bob/friends/123", true, map[string]string{ + "user": "bob", + }), + pt("/user/bob/enemies", false, nil), + }}, +} + +func TestPatterns(t *testing.T) { + t.Parallel() + + for _, pt := range patternTests { + p := pt.pat.Prefix() + if p != pt.prefix { + t.Errorf("Expected prefix %q for %v, got %q", pt.prefix, + pt.pat, p) + } else { + for _, test := range pt.tests { + runTest(t, pt.pat, test) + } + } + } +} + +func runTest(t *testing.T, p Pattern, test patternTest) { + result := p.Match(test.r, test.c) + if result != test.match { + t.Errorf("Expected match(%v, %#v) to return %v", p, + test.r.URL.Path, test.match) + return + } + + if !reflect.DeepEqual(test.c, test.cout) { + t.Errorf("Expected a context of %v, instead got %v", test.cout, + test.c) + } +} From 70edd84d7cf2eebfe48f88ee01559e458e793f06 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 22 Mar 2014 19:45:09 -0700 Subject: [PATCH 012/217] Move patternRe closer to its use --- web/pattern.go | 2 ++ web/router.go | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/pattern.go b/web/pattern.go index f6b11c7..952fc81 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -194,6 +194,8 @@ func (s stringPattern) String() string { return fmt.Sprintf("stringPattern(%q, %v)", s.raw, s.isPrefix) } +var patternRe = regexp.MustCompile(`/:([^/]+)`) + func parseStringPattern(s string, isPrefix bool) stringPattern { matches := patternRe.FindAllStringSubmatchIndex(s, -1) pats := make([]string, len(matches)) diff --git a/web/router.go b/web/router.go index 0db15c4..36d8141 100644 --- a/web/router.go +++ b/web/router.go @@ -65,8 +65,6 @@ type Pattern interface { Match(r *http.Request, c *C) bool } -var patternRe = regexp.MustCompile(`/:([^/]+)`) - func parsePattern(p interface{}, isPrefix bool) Pattern { switch p.(type) { case Pattern: From 4ed96805c3c69722daf3c8d419dd8406165ff9d2 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 22 Mar 2014 21:39:10 -0700 Subject: [PATCH 013/217] Fix old documentation --- web/web.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/web.go b/web/web.go index 60d17bb..8b5e3c1 100644 --- a/web/web.go +++ b/web/web.go @@ -30,11 +30,11 @@ Use your favorite HTTP verbs: Bind parameters using either Sinatra-like patterns or regular expressions: - m.Get("/hello/:name", func(c *web.C, w http.ResponseWriter, r *http.Request) { + m.Get("/hello/:name", func(c web.C, w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Hello, %s!", c.UrlParams["name"]) }) pattern := regexp.MustCompile(`^/ip/(?P(?:\d{1,3}\.){3}\d{1,3})$`) - m.Get(pattern, func(c *web.C, w http.ResponseWriter, r *http.Request) { + m.Get(pattern, func(c web.C, w http.ResponseWriter, r *http.Request) { fmt.Printf(w, "Info for IP address %s:", c.UrlParams["ip"]) }) From 6d67eba9e62038123a8a926cbd6566a6dcb9e9c7 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 22 Mar 2014 21:39:22 -0700 Subject: [PATCH 014/217] Always uppercase the method I have no idea if net/http does this for you already, but I may as well do it myself too... --- web/router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/router.go b/web/router.go index 36d8141..63198af 100644 --- a/web/router.go +++ b/web/router.go @@ -113,7 +113,7 @@ func parseHandler(h interface{}) Handler { } func httpMethod(mname string) method { - switch mname { + switch strings.ToUpper(mname) { case "CONNECT": return mCONNECT case "DELETE": From e21df254bb91ed36a1a7c2fc852da964efbe4663 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 22 Mar 2014 21:40:24 -0700 Subject: [PATCH 015/217] Mux and router tests --- web/mux_test.go | 41 +++++++++++ web/router_test.go | 175 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 web/mux_test.go create mode 100644 web/router_test.go diff --git a/web/mux_test.go b/web/mux_test.go new file mode 100644 index 0000000..4eee7cd --- /dev/null +++ b/web/mux_test.go @@ -0,0 +1,41 @@ +package web + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +// There's... really not a lot to do here. + +func TestIfItWorks(t *testing.T) { + t.Parallel() + + m := New() + ch := make(chan string, 1) + + m.Get("/hello/:name", func(c C, w http.ResponseWriter, r *http.Request) { + greeting := "Hello " + if c.Env != nil { + if g, ok := c.Env["greeting"]; ok { + greeting = g.(string) + } + } + ch <- greeting + c.UrlParams["name"] + }) + + r, _ := http.NewRequest("GET", "/hello/carl", nil) + m.ServeHTTP(httptest.NewRecorder(), r) + out := <-ch + if out != "Hello carl" { + t.Errorf(`Unexpected response %q, expected "Hello carl"`, out) + } + + r, _ = http.NewRequest("GET", "/hello/bob", nil) + env := map[string]interface{}{"greeting": "Yo "} + m.ServeHTTPC(C{Env: env}, httptest.NewRecorder(), r) + out = <-ch + if out != "Yo bob" { + t.Errorf(`Unexpected response %q, expected "Yo bob"`, out) + } +} diff --git a/web/router_test.go b/web/router_test.go new file mode 100644 index 0000000..77a372d --- /dev/null +++ b/web/router_test.go @@ -0,0 +1,175 @@ +package web + +import ( + "net/http" + "net/http/httptest" + "regexp" + "testing" + "time" +) + +// These tests can probably be DRY'd up a bunch + +func makeRouter() *router { + return &router{ + routes: make([]route, 0), + notFound: parseHandler(http.NotFound), + } +} + +func chHandler(ch chan string, s string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ch <- s + }) +} + +var methods = []string{"CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH", + "POST", "PUT", "TRACE", "OTHER"} + +func TestMethods(t *testing.T) { + t.Parallel() + rt := makeRouter() + ch := make(chan string, 1) + + rt.Connect("/", chHandler(ch, "CONNECT")) + rt.Delete("/", chHandler(ch, "DELETE")) + rt.Get("/", chHandler(ch, "GET")) + rt.Head("/", chHandler(ch, "HEAD")) + rt.Options("/", chHandler(ch, "OPTIONS")) + rt.Patch("/", chHandler(ch, "PATCH")) + rt.Post("/", chHandler(ch, "POST")) + rt.Put("/", chHandler(ch, "PUT")) + rt.Trace("/", chHandler(ch, "TRACE")) + rt.Handle("/", chHandler(ch, "OTHER")) + + for _, method := range methods { + r, _ := http.NewRequest(method, "/", nil) + w := httptest.NewRecorder() + rt.route(C{}, w, r) + select { + case val := <-ch: + if val != method { + t.Error("Got %q, expected %q", val, method) + } + case <-time.After(5 * time.Millisecond): + t.Errorf("Timeout waiting for method %q", method) + } + } +} + +type testPattern struct{} + +func (t testPattern) Prefix() string { + return "" +} + +func (t testPattern) Match(r *http.Request, c *C) bool { + return true +} + +func TestPatternTypes(t *testing.T) { + t.Parallel() + rt := makeRouter() + + rt.Get("/hello/carl", http.NotFound) + rt.Get("/hello/:name", http.NotFound) + rt.Get(regexp.MustCompile(`^/hello/(?P.+)$`), http.NotFound) + rt.Get(testPattern{}, http.NotFound) +} + +type testHandler chan string + +func (t testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + t <- "http" +} +func (t testHandler) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { + t <- "httpc" +} + +var testHandlerTable = map[string]string{ + "/a": "http fn", + "/b": "http handler", + "/c": "web fn", + "/d": "web handler", + "/e": "httpc", +} + +func TestHandlerTypes(t *testing.T) { + t.Parallel() + rt := makeRouter() + ch := make(chan string, 1) + + rt.Get("/a", func(w http.ResponseWriter, r *http.Request) { + ch <- "http fn" + }) + rt.Get("/b", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ch <- "http handler" + })) + rt.Get("/c", func(c C, w http.ResponseWriter, r *http.Request) { + ch <- "web fn" + }) + rt.Get("/d", HandlerFunc(func(c C, w http.ResponseWriter, r *http.Request) { + ch <- "web handler" + })) + rt.Get("/e", testHandler(ch)) + + for route, response := range testHandlerTable { + r, _ := http.NewRequest("gEt", route, nil) + w := httptest.NewRecorder() + rt.route(C{}, w, r) + select { + case resp := <-ch: + if resp != response { + t.Errorf("Got %q, expected %q", resp, response) + } + case <-time.After(5 * time.Millisecond): + t.Errorf("Timeout waiting for path %q", route) + } + + } +} + +func TestNotFound(t *testing.T) { + t.Parallel() + rt := makeRouter() + + r, _ := http.NewRequest("post", "/", nil) + w := httptest.NewRecorder() + rt.route(C{}, w, r) + if w.Code != 404 { + t.Errorf("Expected 404, got %d", w.Code) + } + + rt.NotFound(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "I'm a teapot!", http.StatusTeapot) + }) + + r, _ = http.NewRequest("post", "/", nil) + w = httptest.NewRecorder() + rt.route(C{}, w, r) + if w.Code != http.StatusTeapot { + t.Errorf("Expected a teapot, got %d", w.Code) + } +} + +func TestSub(t *testing.T) { + t.Parallel() + rt := makeRouter() + ch := make(chan string, 1) + + rt.Sub("/hello", func(w http.ResponseWriter, r *http.Request) { + ch <- r.URL.Path + }) + + r, _ := http.NewRequest("GET", "/hello/world", nil) + w := httptest.NewRecorder() + rt.route(C{}, w, r) + select { + case val := <-ch: + if val != "/hello/world" { + t.Error("Got %q, expected /hello/world", val) + } + case <-time.After(5 * time.Millisecond): + t.Errorf("Timeout waiting for hello") + } +} From d9925448062cc8350079bc29e73f5a947b748a56 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 22 Mar 2014 22:54:36 -0700 Subject: [PATCH 016/217] Valid HTTP method discovery Package web will now add a key to the environment when it fails to find a valid route for the requested method, but when valid routes exist for other methods. This allows either the 404 handler or a sufficiently clever middleware layer to provide support for OPTIONS automatically. --- web/pattern.go | 10 +++++-- web/pattern_test.go | 2 +- web/router.go | 67 +++++++++++++++++++++++++++++++++++++++++---- web/router_test.go | 52 ++++++++++++++++++++++++++++++++++- 4 files changed, 120 insertions(+), 11 deletions(-) diff --git a/web/pattern.go b/web/pattern.go index 952fc81..4916827 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -19,13 +19,13 @@ type regexpPattern struct { func (p regexpPattern) Prefix() string { return p.prefix } -func (p regexpPattern) Match(r *http.Request, c *C) bool { +func (p regexpPattern) Match(r *http.Request, c *C, dryrun bool) bool { matches := p.re.FindStringSubmatch(r.URL.Path) if matches == nil || len(matches) == 0 { return false } - if len(matches) == 1 { + if c == nil || dryrun || len(matches) == 1 { return true } @@ -149,7 +149,7 @@ func (s stringPattern) Prefix() string { return s.literals[0] } -func (s stringPattern) Match(r *http.Request, c *C) bool { +func (s stringPattern) Match(r *http.Request, c *C, dryrun bool) bool { path := r.URL.Path matches := make([]string, len(s.pats)) for i := 0; i < len(s.pats); i++ { @@ -181,6 +181,10 @@ func (s stringPattern) Match(r *http.Request, c *C) bool { } } + if c == nil || dryrun { + return true + } + if c.UrlParams == nil && len(matches) > 0 { c.UrlParams = make(map[string]string, len(matches)-1) } diff --git a/web/pattern_test.go b/web/pattern_test.go index 7fbe132..b6911d7 100644 --- a/web/pattern_test.go +++ b/web/pattern_test.go @@ -149,7 +149,7 @@ func TestPatterns(t *testing.T) { } func runTest(t *testing.T, p Pattern, test patternTest) { - result := p.Match(test.r, test.c) + result := p.Match(test.r, test.c, false) if result != test.match { t.Errorf("Expected match(%v, %#v) to return %v", p, test.r.URL.Path, test.match) diff --git a/web/router.go b/web/router.go index 63198af..f3f63ae 100644 --- a/web/router.go +++ b/web/router.go @@ -28,6 +28,8 @@ const ( mPOST | mPUT | mTRACE | mIDK ) +const validMethods = "goji.web.validMethods" + type route struct { // Theory: most real world routes have a string prefix which is both // cheap(-ish) to test against and pretty selective. And, conveniently, @@ -61,8 +63,9 @@ type Pattern interface { // Returns true if the request satisfies the pattern. This function is // free to examine both the request and the context to make this // decision. After it is certain that the request matches, this function - // should mutate or create c.UrlParams if necessary. - Match(r *http.Request, c *C) bool + // should mutate or create c.UrlParams if necessary, unless dryrun is + // set. + Match(r *http.Request, c *C, dryrun bool) bool } func parsePattern(p interface{}, isPrefix bool) Pattern { @@ -139,16 +142,64 @@ func httpMethod(mname string) method { func (rt *router) route(c C, w http.ResponseWriter, r *http.Request) { m := httpMethod(r.Method) + var methods method for _, route := range rt.routes { - if route.method&m == 0 || - !strings.HasPrefix(r.URL.Path, route.prefix) || - !route.pattern.Match(r, &c) { + if !strings.HasPrefix(r.URL.Path, route.prefix) || + !route.pattern.Match(r, &c, false) { + continue } - route.handler.ServeHTTPC(c, w, r) + + if route.method&m != 0 { + route.handler.ServeHTTPC(c, w, r) + return + } else if route.pattern.Match(r, &c, true) { + methods |= route.method + } + } + + if methods == 0 { + rt.notFound.ServeHTTPC(c, w, r) return } + // Oh god kill me now + var methodsList = make([]string, 0) + if methods&mCONNECT != 0 { + methodsList = append(methodsList, "CONNECT") + } + if methods&mDELETE != 0 { + methodsList = append(methodsList, "DELETE") + } + if methods&mGET != 0 { + methodsList = append(methodsList, "GET") + } + if methods&mHEAD != 0 { + methodsList = append(methodsList, "HEAD") + } + if methods&mOPTIONS != 0 { + methodsList = append(methodsList, "OPTIONS") + } + if methods&mPATCH != 0 { + methodsList = append(methodsList, "PATCH") + } + if methods&mPOST != 0 { + methodsList = append(methodsList, "POST") + } + if methods&mPUT != 0 { + methodsList = append(methodsList, "PUT") + } + if methods&mTRACE != 0 { + methodsList = append(methodsList, "TRACE") + } + + if c.Env == nil { + c.Env = map[string]interface{}{ + validMethods: methodsList, + } + } else { + c.Env[validMethods] = methodsList + } rt.notFound.ServeHTTPC(c, w, r) } @@ -270,6 +321,10 @@ func (m *router) Sub(pattern string, handler interface{}) { // Set the fallback (i.e., 404) handler for this mux. See the documentation for // type Mux for a description of what types are accepted for handler. +// +// As a convenience, the environment variable "goji.web.validMethods" will be +// set to the list of HTTP methods that could have been routed had they been +// provided on an otherwise identical request func (m *router) NotFound(handler interface{}) { m.notFound = parseHandler(handler) } diff --git a/web/router_test.go b/web/router_test.go index 77a372d..44331ae 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -3,6 +3,7 @@ package web import ( "net/http" "net/http/httptest" + "reflect" "regexp" "testing" "time" @@ -63,10 +64,12 @@ func (t testPattern) Prefix() string { return "" } -func (t testPattern) Match(r *http.Request, c *C) bool { +func (t testPattern) Match(r *http.Request, c *C, dryrun bool) bool { return true } +var _ Pattern = testPattern{} + func TestPatternTypes(t *testing.T) { t.Parallel() rt := makeRouter() @@ -173,3 +176,50 @@ func TestSub(t *testing.T) { t.Errorf("Timeout waiting for hello") } } + +var validMethodsTable = map[string][]string{ + "/hello/carl": {"DELETE", "GET", "PATCH", "POST", "PUT"}, + "/hello/bob": {"DELETE", "GET", "HEAD", "PATCH", "PUT"}, + "/hola/carl": {"DELETE", "GET", "PUT"}, + "/hola/bob": {"DELETE"}, + "/does/not/compute": {}, +} + +func TestValidMethods(t *testing.T) { + t.Parallel() + rt := makeRouter() + ch := make(chan []string, 1) + + rt.NotFound(func(c C, w http.ResponseWriter, r *http.Request) { + if c.Env == nil { + ch <- []string{} + return + } + methods, ok := c.Env[validMethods] + if !ok { + ch <- []string{} + return + } + ch <- methods.([]string) + }) + + rt.Get("/hello/carl", http.NotFound) + rt.Post("/hello/carl", http.NotFound) + rt.Head("/hello/bob", http.NotFound) + rt.Get("/hello/:name", http.NotFound) + rt.Put("/hello/:name", http.NotFound) + rt.Patch("/hello/:name", http.NotFound) + rt.Get("/:greet/carl", http.NotFound) + rt.Put("/:greet/carl", http.NotFound) + rt.Delete("/:greet/:anyone", http.NotFound) + + for path, eMethods := range validMethodsTable { + r, _ := http.NewRequest("BOGUS", path, nil) + rt.route(C{}, httptest.NewRecorder(), r) + aMethods := <-ch + if !reflect.DeepEqual(eMethods, aMethods) { + t.Errorf("For %q, expected %v, got %v", path, eMethods, + aMethods) + } + } +} From d78a6852889a6d919ef89973ed848694fc9a5cb5 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 22 Mar 2014 23:03:13 -0700 Subject: [PATCH 017/217] DRY up methods map --- web/router.go | 69 ++++++++++++++++----------------------------------- 1 file changed, 21 insertions(+), 48 deletions(-) diff --git a/web/router.go b/web/router.go index f3f63ae..f7b9de4 100644 --- a/web/router.go +++ b/web/router.go @@ -4,6 +4,7 @@ import ( "log" "net/http" "regexp" + "sort" "strings" "sync" ) @@ -30,6 +31,18 @@ const ( const validMethods = "goji.web.validMethods" +var validMethodsMap = map[string]method{ + "CONNECT": mCONNECT, + "DELETE": mDELETE, + "GET": mGET, + "HEAD": mHEAD, + "OPTIONS": mOPTIONS, + "PATCH": mPATCH, + "POST": mPOST, + "PUT": mPUT, + "TRACE": mTRACE, +} + type route struct { // Theory: most real world routes have a string prefix which is both // cheap(-ish) to test against and pretty selective. And, conveniently, @@ -116,28 +129,10 @@ func parseHandler(h interface{}) Handler { } func httpMethod(mname string) method { - switch strings.ToUpper(mname) { - case "CONNECT": - return mCONNECT - case "DELETE": - return mDELETE - case "GET": - return mGET - case "HEAD": - return mHEAD - case "OPTIONS": - return mOPTIONS - case "PATCH": - return mPATCH - case "POST": - return mPOST - case "PUT": - return mPUT - case "TRACE": - return mTRACE - default: - return mIDK + if method, ok := validMethodsMap[strings.ToUpper(mname)]; ok { + return method } + return mIDK } func (rt *router) route(c C, w http.ResponseWriter, r *http.Request) { @@ -163,35 +158,13 @@ func (rt *router) route(c C, w http.ResponseWriter, r *http.Request) { return } - // Oh god kill me now var methodsList = make([]string, 0) - if methods&mCONNECT != 0 { - methodsList = append(methodsList, "CONNECT") - } - if methods&mDELETE != 0 { - methodsList = append(methodsList, "DELETE") - } - if methods&mGET != 0 { - methodsList = append(methodsList, "GET") - } - if methods&mHEAD != 0 { - methodsList = append(methodsList, "HEAD") - } - if methods&mOPTIONS != 0 { - methodsList = append(methodsList, "OPTIONS") - } - if methods&mPATCH != 0 { - methodsList = append(methodsList, "PATCH") - } - if methods&mPOST != 0 { - methodsList = append(methodsList, "POST") - } - if methods&mPUT != 0 { - methodsList = append(methodsList, "PUT") - } - if methods&mTRACE != 0 { - methodsList = append(methodsList, "TRACE") + for mname, meth := range validMethodsMap { + if methods&meth != 0 { + methodsList = append(methodsList, mname) + } } + sort.Strings(methodsList) if c.Env == nil { c.Env = map[string]interface{}{ From f9808345b9922f0c90bf7df00a1f6fc28b404663 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 02:58:24 -0700 Subject: [PATCH 018/217] Standard middlewares I've currently written three: - Request ID generation - Request logging, with color (!) - Recovery from panics --- README.md | 18 ----- default.go | 5 ++ goji.go | 2 + web/middleware/logger.go | 149 +++++++++++++++++++++++++++++++++++ web/middleware/middleware.go | 4 + web/middleware/recoverer.go | 44 +++++++++++ web/middleware/request_id.go | 70 ++++++++++++++++ web/middleware/terminal.go | 44 +++++++++++ 8 files changed, 318 insertions(+), 18 deletions(-) create mode 100644 web/middleware/logger.go create mode 100644 web/middleware/middleware.go create mode 100644 web/middleware/recoverer.go create mode 100644 web/middleware/request_id.go create mode 100644 web/middleware/terminal.go diff --git a/README.md b/README.md index b0df83e..9a1affb 100644 --- a/README.md +++ b/README.md @@ -47,21 +47,3 @@ Features [graceful]: http://godoc.org/github.com/zenazn/goji/graceful [param]: http://godoc.org/github.com/zenazn/goji/param - -Todo ----- - -Goji probably deserves a bit more love before anyone actually tries to use it. -Things that need doing include: - -* Support for omitting trailing slashes on routes which include them. -* Tests for `goji/web`. There are currently no tests. This probably means - `goji/web` is made of bugs. I'm sorry. -* Standard middleware implementations. I'm currently thinking: - * Request ID assigner: injects a request ID into the environment. - * Request logger: logs requests as they come in. Preferrably with request IDs - and maybe even with colors. - * Request timer: maybe part of the request logger. Time how long each request - takes and print it. Maybe with color. - * Error handler: recovers from panics, does something nice with the output. - * Healthcheck endpoint: Always returns OK. diff --git a/default.go b/default.go index c43cccf..fca2a27 100644 --- a/default.go +++ b/default.go @@ -2,6 +2,7 @@ package goji import ( "github.com/zenazn/goji/web" + "github.com/zenazn/goji/web/middleware" ) // The default web.Mux. @@ -9,6 +10,10 @@ var DefaultMux *web.Mux func init() { DefaultMux = web.New() + + DefaultMux.Use("RequestId", middleware.RequestId) + DefaultMux.Use("Logger", middleware.Logger) + DefaultMux.Use("Recoverer", middleware.Recoverer) } // Append the given middleware to the default Mux's middleware stack. See the diff --git a/goji.go b/goji.go index e973f7c..8f68ac3 100644 --- a/goji.go +++ b/goji.go @@ -48,6 +48,8 @@ func Serve() { flag.Parse() } + log.SetFlags(log.Flags() | log.Lmicroseconds) + // Install our handler at the root of the standard net/http default mux. // This allows packages like expvar to continue working as expected. http.Handle("/", DefaultMux) diff --git a/web/middleware/logger.go b/web/middleware/logger.go new file mode 100644 index 0000000..72c4115 --- /dev/null +++ b/web/middleware/logger.go @@ -0,0 +1,149 @@ +package middleware + +import ( + "bufio" + "bytes" + "log" + "net" + "net/http" + "time" + + "github.com/zenazn/goji/web" +) + +// Logger is a middleware that logs the start and end of each request, along +// with some useful data about what was requested, what the response status was, +// and how long it took to return. When standard output is a TTY, Logger will +// print in color, otherwise it will print in black and white. +// +// Logger prints a request ID if one is provided. +func Logger(c *web.C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + reqId := GetReqId(*c) + + printStart(reqId, r) + + lw := wrapWriter(w) + + t1 := time.Now() + h.ServeHTTP(lw, r) + lw.maybeWriteHeader() + t2 := time.Now() + + printEnd(reqId, lw, t2.Sub(t1)) + } + + return http.HandlerFunc(fn) +} + +func printStart(reqId string, r *http.Request) { + var buf bytes.Buffer + + if reqId != "" { + cW(&buf, bBlack, "[%s] ", reqId) + } + buf.WriteString("Started ") + cW(&buf, bMagenta, "%s ", r.Method) + cW(&buf, nBlue, "%q ", r.URL.String()) + buf.WriteString("from ") + buf.WriteString(r.RemoteAddr) + + log.Print(buf.String()) +} + +func printEnd(reqId string, w writerProxy, dt time.Duration) { + var buf bytes.Buffer + + if reqId != "" { + cW(&buf, bBlack, "[%s] ", reqId) + } + buf.WriteString("Returning ") + if w.status() < 200 { + cW(&buf, bBlue, "%03d", w.status()) + } else if w.status() < 300 { + cW(&buf, bGreen, "%03d", w.status()) + } else if w.status() < 400 { + cW(&buf, bCyan, "%03d", w.status()) + } else if w.status() < 500 { + cW(&buf, bYellow, "%03d", w.status()) + } else { + cW(&buf, bRed, "%03d", w.status()) + } + buf.WriteString(" in ") + if dt < 500*time.Millisecond { + cW(&buf, nGreen, "%s", dt) + } else if dt < 5*time.Second { + cW(&buf, nYellow, "%s", dt) + } else { + cW(&buf, nRed, "%s", dt) + } + + log.Print(buf.String()) +} + +func wrapWriter(w http.ResponseWriter) writerProxy { + _, cn := w.(http.CloseNotifier) + _, fl := w.(http.Flusher) + _, hj := w.(http.Hijacker) + + bw := basicWriter{ResponseWriter: w} + if cn && fl && hj { + return &fancyWriter{bw} + } else { + return &bw + } +} + +type writerProxy interface { + http.ResponseWriter + maybeWriteHeader() + status() int +} + +type basicWriter struct { + http.ResponseWriter + wroteHeader bool + code int +} + +func (b *basicWriter) WriteHeader(code int) { + b.code = code + b.wroteHeader = true + b.ResponseWriter.WriteHeader(code) +} +func (b *basicWriter) Write(buf []byte) (int, error) { + b.maybeWriteHeader() + return b.ResponseWriter.Write(buf) +} +func (b *basicWriter) maybeWriteHeader() { + if !b.wroteHeader { + b.WriteHeader(http.StatusOK) + } +} +func (b *basicWriter) status() int { + return b.code +} +func (b *basicWriter) Unwrap() http.ResponseWriter { + return b.ResponseWriter +} + +type fancyWriter struct { + basicWriter +} + +func (f *fancyWriter) CloseNotify() <-chan bool { + cn := f.basicWriter.ResponseWriter.(http.CloseNotifier) + return cn.CloseNotify() +} +func (f *fancyWriter) Flush() { + fl := f.basicWriter.ResponseWriter.(http.Flusher) + fl.Flush() +} +func (f *fancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hj := f.basicWriter.ResponseWriter.(http.Hijacker) + return hj.Hijack() +} + +var _ http.CloseNotifier = &fancyWriter{} +var _ http.Flusher = &fancyWriter{} +var _ http.Hijacker = &fancyWriter{} diff --git a/web/middleware/middleware.go b/web/middleware/middleware.go new file mode 100644 index 0000000..23cfde2 --- /dev/null +++ b/web/middleware/middleware.go @@ -0,0 +1,4 @@ +/* +Package middleware provides several standard middleware implementations. +*/ +package middleware diff --git a/web/middleware/recoverer.go b/web/middleware/recoverer.go new file mode 100644 index 0000000..adf1ca4 --- /dev/null +++ b/web/middleware/recoverer.go @@ -0,0 +1,44 @@ +package middleware + +import ( + "bytes" + "log" + "net/http" + "runtime/debug" + + "github.com/zenazn/goji/web" +) + +// Recoverer is a middleware that recovers from panics, logs the panic (and a +// backtrace), and returns a HTTP 500 (Internal Server Error) status if +// possible. +// +// Recoverer prints a request ID if one is provided. +func Recoverer(c *web.C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + reqId := GetReqId(*c) + + defer func() { + if err := recover(); err != nil { + printPanic(reqId, err) + debug.PrintStack() + http.Error(w, http.StatusText(500), 500) + } + }() + + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} + +func printPanic(reqId string, err interface{}) { + var buf bytes.Buffer + + if reqId != "" { + cW(&buf, bBlack, "[%s] ", reqId) + } + cW(&buf, bRed, "panic: %#v", err) + + log.Print(buf.String()) +} diff --git a/web/middleware/request_id.go b/web/middleware/request_id.go new file mode 100644 index 0000000..7c8f5d7 --- /dev/null +++ b/web/middleware/request_id.go @@ -0,0 +1,70 @@ +package middleware + +import ( + "crypto/rand" + "encoding/base64" + "fmt" + "net/http" + "os" + "strings" + "sync/atomic" + + "github.com/zenazn/goji/web" +) + +// Key to use when setting the request ID. +const RequestIdKey = "reqId" + +var prefix string +var reqid uint64 + +func init() { + hostname, err := os.Hostname() + if hostname == "" || err != nil { + hostname = "localhost" + } + var buf [12]byte + rand.Read(buf[:]) + b64 := base64.StdEncoding.EncodeToString(buf[:]) + // Strip out annoying characters. We have something like a billion to + // one chance of having enough from 12 bytes of entropy + b64 = strings.NewReplacer("+", "", "/", "").Replace(b64) + + prefix = fmt.Sprintf("%s/%s", hostname, b64[0:8]) +} + +// RequestId is a middleware that injects a request ID into the context of each +// request. A request ID is a string of the form "host.example.com/random-0001", +// where "random" is a base62 random string that uniquely identifies this go +// process, and where the last number is an atomically incremented request +// counter. +func RequestId(c *web.C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if c.Env == nil { + c.Env = make(map[string]interface{}) + } + myid := atomic.AddUint64(&reqid, 1) + c.Env[RequestIdKey] = fmt.Sprintf("%s-%06d", prefix, myid) + + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} + +// Get a request ID from the given context if one is present. Returns the empty +// string if a request ID cannot be found. +func GetReqId(c web.C) string { + if c.Env == nil { + return "" + } + v, ok := c.Env[RequestIdKey] + if !ok { + return "" + } + if reqId, ok := v.(string); ok { + return reqId + } else { + return "" + } +} diff --git a/web/middleware/terminal.go b/web/middleware/terminal.go new file mode 100644 index 0000000..813e83d --- /dev/null +++ b/web/middleware/terminal.go @@ -0,0 +1,44 @@ +package middleware + +import ( + "bytes" + "fmt" + + "code.google.com/p/go.crypto/ssh/terminal" +) + +var ( + // Normal colors + nBlack = []byte{'\033', '[', '3', '0', 'm'} + nRed = []byte{'\033', '[', '3', '1', 'm'} + nGreen = []byte{'\033', '[', '3', '2', 'm'} + nYellow = []byte{'\033', '[', '3', '3', 'm'} + nBlue = []byte{'\033', '[', '3', '4', 'm'} + nMagenta = []byte{'\033', '[', '3', '5', 'm'} + nCyan = []byte{'\033', '[', '3', '6', 'm'} + nWhite = []byte{'\033', '[', '3', '7', 'm'} + // Bright colors + bBlack = []byte{'\033', '[', '3', '0', ';', '1', 'm'} + bRed = []byte{'\033', '[', '3', '1', ';', '1', 'm'} + bGreen = []byte{'\033', '[', '3', '2', ';', '1', 'm'} + bYellow = []byte{'\033', '[', '3', '3', ';', '1', 'm'} + bBlue = []byte{'\033', '[', '3', '4', ';', '1', 'm'} + bMagenta = []byte{'\033', '[', '3', '5', ';', '1', 'm'} + bCyan = []byte{'\033', '[', '3', '6', ';', '1', 'm'} + bWhite = []byte{'\033', '[', '3', '7', ';', '1', 'm'} + + reset = []byte{'\033', '[', '0', 'm'} +) + +var isTTY = terminal.IsTerminal(1) + +// colorWrite +func cW(buf *bytes.Buffer, color []byte, s string, args ...interface{}) { + if isTTY { + buf.Write(color) + } + fmt.Fprintf(buf, s, args...) + if isTTY { + buf.Write(reset) + } +} From 66431cbd3b79e52baa8cc87101d165d92d8453d7 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 03:28:57 -0700 Subject: [PATCH 019/217] Move writer proxy to a separate file --- web/middleware/logger.go | 56 ------------------------------- web/middleware/writer_proxy.go | 61 ++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 56 deletions(-) create mode 100644 web/middleware/writer_proxy.go diff --git a/web/middleware/logger.go b/web/middleware/logger.go index 72c4115..b8d590a 100644 --- a/web/middleware/logger.go +++ b/web/middleware/logger.go @@ -1,10 +1,8 @@ package middleware import ( - "bufio" "bytes" "log" - "net" "net/http" "time" @@ -93,57 +91,3 @@ func wrapWriter(w http.ResponseWriter) writerProxy { return &bw } } - -type writerProxy interface { - http.ResponseWriter - maybeWriteHeader() - status() int -} - -type basicWriter struct { - http.ResponseWriter - wroteHeader bool - code int -} - -func (b *basicWriter) WriteHeader(code int) { - b.code = code - b.wroteHeader = true - b.ResponseWriter.WriteHeader(code) -} -func (b *basicWriter) Write(buf []byte) (int, error) { - b.maybeWriteHeader() - return b.ResponseWriter.Write(buf) -} -func (b *basicWriter) maybeWriteHeader() { - if !b.wroteHeader { - b.WriteHeader(http.StatusOK) - } -} -func (b *basicWriter) status() int { - return b.code -} -func (b *basicWriter) Unwrap() http.ResponseWriter { - return b.ResponseWriter -} - -type fancyWriter struct { - basicWriter -} - -func (f *fancyWriter) CloseNotify() <-chan bool { - cn := f.basicWriter.ResponseWriter.(http.CloseNotifier) - return cn.CloseNotify() -} -func (f *fancyWriter) Flush() { - fl := f.basicWriter.ResponseWriter.(http.Flusher) - fl.Flush() -} -func (f *fancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { - hj := f.basicWriter.ResponseWriter.(http.Hijacker) - return hj.Hijack() -} - -var _ http.CloseNotifier = &fancyWriter{} -var _ http.Flusher = &fancyWriter{} -var _ http.Hijacker = &fancyWriter{} diff --git a/web/middleware/writer_proxy.go b/web/middleware/writer_proxy.go new file mode 100644 index 0000000..634bc4e --- /dev/null +++ b/web/middleware/writer_proxy.go @@ -0,0 +1,61 @@ +package middleware + +import ( + "bufio" + "net" + "net/http" +) + +type writerProxy interface { + http.ResponseWriter + maybeWriteHeader() + status() int +} + +type basicWriter struct { + http.ResponseWriter + wroteHeader bool + code int +} + +func (b *basicWriter) WriteHeader(code int) { + b.code = code + b.wroteHeader = true + b.ResponseWriter.WriteHeader(code) +} +func (b *basicWriter) Write(buf []byte) (int, error) { + b.maybeWriteHeader() + return b.ResponseWriter.Write(buf) +} +func (b *basicWriter) maybeWriteHeader() { + if !b.wroteHeader { + b.WriteHeader(http.StatusOK) + } +} +func (b *basicWriter) status() int { + return b.code +} +func (b *basicWriter) Unwrap() http.ResponseWriter { + return b.ResponseWriter +} + +type fancyWriter struct { + basicWriter +} + +func (f *fancyWriter) CloseNotify() <-chan bool { + cn := f.basicWriter.ResponseWriter.(http.CloseNotifier) + return cn.CloseNotify() +} +func (f *fancyWriter) Flush() { + fl := f.basicWriter.ResponseWriter.(http.Flusher) + fl.Flush() +} +func (f *fancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hj := f.basicWriter.ResponseWriter.(http.Hijacker) + return hj.Hijack() +} + +var _ http.CloseNotifier = &fancyWriter{} +var _ http.Flusher = &fancyWriter{} +var _ http.Hijacker = &fancyWriter{} From 9e5ef71c04a299088512e5686fb35608307c4658 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 03:31:46 -0700 Subject: [PATCH 020/217] Automatic OPTIONS middleware --- default.go | 1 + web/middleware/options.go | 63 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 web/middleware/options.go diff --git a/default.go b/default.go index fca2a27..6ec68ed 100644 --- a/default.go +++ b/default.go @@ -14,6 +14,7 @@ func init() { DefaultMux.Use("RequestId", middleware.RequestId) DefaultMux.Use("Logger", middleware.Logger) DefaultMux.Use("Recoverer", middleware.Recoverer) + DefaultMux.Use("AutomaticOptions", middleware.AutomaticOptions) } // Append the given middleware to the default Mux's middleware stack. See the diff --git a/web/middleware/options.go b/web/middleware/options.go new file mode 100644 index 0000000..ffa969b --- /dev/null +++ b/web/middleware/options.go @@ -0,0 +1,63 @@ +package middleware + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + + "github.com/zenazn/goji/web" +) + +// Automatically return an appropriate "Allow" header when the request method is +// OPTIONS and the request would have otherwise been 404'd. +func AutomaticOptions(c *web.C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + // This will probably slow down OPTIONS calls a bunch, but it + // probably won't happen too much, and it'll just be hitting the + // 404 route anyways. + var fw *httptest.ResponseRecorder + pw := w + if strings.ToUpper(r.Method) == "OPTIONS" { + fw = httptest.NewRecorder() + pw = fw + } + + h.ServeHTTP(pw, r) + + if fw == nil { + return + } + + for k, v := range fw.Header() { + w.Header()[k] = v + } + + methods := getValidMethods(*c) + + if fw.Code == http.StatusNotFound && methods != nil { + w.Header().Set("Allow", strings.Join(methods, ", ")) + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(fw.Code) + io.Copy(w, fw.Body) + } + } + + return http.HandlerFunc(fn) +} + +func getValidMethods(c web.C) []string { + if c.Env == nil { + return nil + } + v, ok := c.Env["goji.web.validMethods"] + if !ok { + return nil + } + if methods, ok := v.([]string); ok { + return methods + } else { + return nil + } +} From 232a1ca725943e7fc6a4832e133b79a99a2ae079 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 12:01:56 -0700 Subject: [PATCH 021/217] Replace terminal dependency with hand-waving In order to avoid a dependency on the go.crypto terminal package, let's try to do our own TTY sniffing. I think in practice this will work surprisingly well, even if it feels incredibly sketchy. --- web/middleware/terminal.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/web/middleware/terminal.go b/web/middleware/terminal.go index 813e83d..db02917 100644 --- a/web/middleware/terminal.go +++ b/web/middleware/terminal.go @@ -3,8 +3,7 @@ package middleware import ( "bytes" "fmt" - - "code.google.com/p/go.crypto/ssh/terminal" + "os" ) var ( @@ -30,7 +29,24 @@ var ( reset = []byte{'\033', '[', '0', 'm'} ) -var isTTY = terminal.IsTerminal(1) +var isTTY bool + +func init() { + // This is sort of cheating: if stdout is a character device, we assume + // that means it's a TTY. Unfortunately, there are many non-TTY + // character devices, but fortunately stdout is rarely set to any of + // them. + // + // We could solve this properly by pulling in a dependency on + // code.google.com/p/go.crypto/ssh/terminal, for instance, but as a + // heuristic for whether to print in color or in black-and-white, I'd + // really rather not. + fi, err := os.Stdout.Stat() + if err == nil { + m := os.ModeDevice | os.ModeCharDevice + isTTY = fi.Mode()&m == m + } +} // colorWrite func cW(buf *bytes.Buffer, color []byte, s string, args ...interface{}) { From 836cb84fdcabd74b8f4739ea238bfd85e9af4374 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 12:50:04 -0700 Subject: [PATCH 022/217] Automatic HEAD support for GET handlers --- web/router.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/web/router.go b/web/router.go index f7b9de4..02315da 100644 --- a/web/router.go +++ b/web/router.go @@ -223,8 +223,13 @@ func (m *router) Delete(pattern interface{}, handler interface{}) { // Dispatch to the given handler when the pattern matches and the HTTP method is // GET. See the documentation for type Mux for a description of what types are // accepted for pattern and handler. +// +// All GET handlers also transparently serve HEAD requests, since net/http will +// take care of all the fiddly bits for you. If you wish to provide an alternate +// implementation of HEAD, you should add a handler explicitly and place it +// above your GET handler. func (m *router) Get(pattern interface{}, handler interface{}) { - m.handleUntyped(pattern, mGET, handler) + m.handleUntyped(pattern, mGET|mHEAD, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is From 9641eb21e384af0e7b851484210697f9748fd1c2 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 12:51:21 -0700 Subject: [PATCH 023/217] Overwrite Connection: close header, don't append --- graceful/middleware.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graceful/middleware.go b/graceful/middleware.go index 34a3368..fedcf37 100644 --- a/graceful/middleware.go +++ b/graceful/middleware.go @@ -59,7 +59,7 @@ func (b *basicWriter) maybeClose() { b.headerWritten = true select { case <-kill: - b.ResponseWriter.Header().Add("Connection", "close") + b.ResponseWriter.Header().Set("Connection", "close") default: } } From fb3ce04ee472da00fffcd61f72e8ad40c884359c Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 12:53:31 -0700 Subject: [PATCH 024/217] Add OPTIONS to the list of allowed methods --- web/middleware/options.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/web/middleware/options.go b/web/middleware/options.go index ffa969b..c8ed63c 100644 --- a/web/middleware/options.go +++ b/web/middleware/options.go @@ -36,6 +36,7 @@ func AutomaticOptions(c *web.C, h http.Handler) http.Handler { methods := getValidMethods(*c) if fw.Code == http.StatusNotFound && methods != nil { + methods = addMethod(methods, "OPTIONS") w.Header().Set("Allow", strings.Join(methods, ", ")) w.WriteHeader(http.StatusOK) } else { @@ -61,3 +62,14 @@ func getValidMethods(c web.C) []string { return nil } } + +// Assumption: the list of methods is teensy, and that anything we could +// possibly want to do here is going to be fast. +func addMethod(methods []string, method string) []string { + for _, m := range methods { + if m == method { + return methods + } + } + return append(methods, method) +} From 05e53fc481095b74fd8c53445f3af6e945de3809 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 13:00:21 -0700 Subject: [PATCH 025/217] fancyWriters should also implement io.ReaderFrom For sendfile(2) support. I should really DRY up the implementations here... --- graceful/middleware.go | 16 +++++++++++++++- web/middleware/logger.go | 13 ------------- web/middleware/writer_proxy.go | 21 +++++++++++++++++++++ 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/graceful/middleware.go b/graceful/middleware.go index fedcf37..7f4683f 100644 --- a/graceful/middleware.go +++ b/graceful/middleware.go @@ -2,6 +2,7 @@ package graceful import ( "bufio" + "io" "net" "net/http" ) @@ -36,10 +37,11 @@ func Middleware(h http.Handler) http.Handler { _, cn := w.(http.CloseNotifier) _, fl := w.(http.Flusher) _, hj := w.(http.Hijacker) + _, rf := w.(io.ReaderFrom) bw := basicWriter{ResponseWriter: w} - if cn && fl && hj { + if cn && fl && hj && rf { h.ServeHTTP(&fancyWriter{bw}, r) } else { h.ServeHTTP(&bw, r) @@ -104,3 +106,15 @@ func (f *fancyWriter) Hijack() (c net.Conn, b *bufio.ReadWriter, e error) { return } +func (f *fancyWriter) ReadFrom(r io.Reader) (int64, error) { + rf := f.basicWriter.ResponseWriter.(io.ReaderFrom) + if !f.basicWriter.headerWritten { + f.basicWriter.maybeClose() + } + return rf.ReadFrom(r) +} + +var _ http.CloseNotifier = &fancyWriter{} +var _ http.Flusher = &fancyWriter{} +var _ http.Hijacker = &fancyWriter{} +var _ io.ReaderFrom = &fancyWriter{} diff --git a/web/middleware/logger.go b/web/middleware/logger.go index b8d590a..4fbb0e9 100644 --- a/web/middleware/logger.go +++ b/web/middleware/logger.go @@ -78,16 +78,3 @@ func printEnd(reqId string, w writerProxy, dt time.Duration) { log.Print(buf.String()) } - -func wrapWriter(w http.ResponseWriter) writerProxy { - _, cn := w.(http.CloseNotifier) - _, fl := w.(http.Flusher) - _, hj := w.(http.Hijacker) - - bw := basicWriter{ResponseWriter: w} - if cn && fl && hj { - return &fancyWriter{bw} - } else { - return &bw - } -} diff --git a/web/middleware/writer_proxy.go b/web/middleware/writer_proxy.go index 634bc4e..f9cfe94 100644 --- a/web/middleware/writer_proxy.go +++ b/web/middleware/writer_proxy.go @@ -2,10 +2,25 @@ package middleware import ( "bufio" + "io" "net" "net/http" ) +func wrapWriter(w http.ResponseWriter) writerProxy { + _, cn := w.(http.CloseNotifier) + _, fl := w.(http.Flusher) + _, hj := w.(http.Hijacker) + _, rf := w.(io.ReaderFrom) + + bw := basicWriter{ResponseWriter: w} + if cn && fl && hj && rf { + return &fancyWriter{bw} + } else { + return &bw + } +} + type writerProxy interface { http.ResponseWriter maybeWriteHeader() @@ -55,7 +70,13 @@ func (f *fancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { hj := f.basicWriter.ResponseWriter.(http.Hijacker) return hj.Hijack() } +func (f *fancyWriter) ReadFrom(r io.Reader) (int64, error) { + rf := f.basicWriter.ResponseWriter.(io.ReaderFrom) + f.basicWriter.maybeWriteHeader() + return rf.ReadFrom(r) +} var _ http.CloseNotifier = &fancyWriter{} var _ http.Flusher = &fancyWriter{} var _ http.Hijacker = &fancyWriter{} +var _ io.ReaderFrom = &fancyWriter{} From 386323a6edcd74962da674f36df8489f5b1fbe4b Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 13:02:45 -0700 Subject: [PATCH 026/217] HTTP method names are case sensitive Who knew! (RFC 2616, 5.1.1) --- web/middleware/options.go | 2 +- web/router.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/middleware/options.go b/web/middleware/options.go index c8ed63c..d267ca9 100644 --- a/web/middleware/options.go +++ b/web/middleware/options.go @@ -18,7 +18,7 @@ func AutomaticOptions(c *web.C, h http.Handler) http.Handler { // 404 route anyways. var fw *httptest.ResponseRecorder pw := w - if strings.ToUpper(r.Method) == "OPTIONS" { + if r.Method == "OPTIONS" { fw = httptest.NewRecorder() pw = fw } diff --git a/web/router.go b/web/router.go index 02315da..86041ef 100644 --- a/web/router.go +++ b/web/router.go @@ -129,7 +129,7 @@ func parseHandler(h interface{}) Handler { } func httpMethod(mname string) method { - if method, ok := validMethodsMap[strings.ToUpper(mname)]; ok { + if method, ok := validMethodsMap[mname]; ok { return method } return mIDK From aca58b22cfe17db7c5fc9a94c1ab647c6783cd08 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 14:11:11 -0700 Subject: [PATCH 027/217] Make ValidMethodsKey public --- web/middleware/options.go | 2 +- web/router.go | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/web/middleware/options.go b/web/middleware/options.go index d267ca9..07437c2 100644 --- a/web/middleware/options.go +++ b/web/middleware/options.go @@ -52,7 +52,7 @@ func getValidMethods(c web.C) []string { if c.Env == nil { return nil } - v, ok := c.Env["goji.web.validMethods"] + v, ok := c.Env[web.ValidMethodsKey] if !ok { return nil } diff --git a/web/router.go b/web/router.go index 86041ef..770f2d9 100644 --- a/web/router.go +++ b/web/router.go @@ -29,7 +29,9 @@ const ( mPOST | mPUT | mTRACE | mIDK ) -const validMethods = "goji.web.validMethods" +// The key used to communicate to the NotFound handler what methods would have +// been allowed if they'd been provided. +const ValidMethodsKey = "goji.web.validMethods" var validMethodsMap = map[string]method{ "CONNECT": mCONNECT, @@ -168,10 +170,10 @@ func (rt *router) route(c C, w http.ResponseWriter, r *http.Request) { if c.Env == nil { c.Env = map[string]interface{}{ - validMethods: methodsList, + ValidMethodsKey: methodsList, } } else { - c.Env[validMethods] = methodsList + c.Env[ValidMethodsKey] = methodsList } rt.notFound.ServeHTTPC(c, w, r) } @@ -300,9 +302,10 @@ func (m *router) Sub(pattern string, handler interface{}) { // Set the fallback (i.e., 404) handler for this mux. See the documentation for // type Mux for a description of what types are accepted for handler. // -// As a convenience, the environment variable "goji.web.validMethods" will be -// set to the list of HTTP methods that could have been routed had they been -// provided on an otherwise identical request +// As a convenience, the context environment variable "goji.web.validMethods" +// (also available as the constant ValidMethodsKey) will be set to the list of +// HTTP methods that could have been routed had they been provided on an +// otherwise identical request. func (m *router) NotFound(handler interface{}) { m.notFound = parseHandler(handler) } From 92b35046d28c386b910e86ab4601e60f551bc6f0 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 14:15:55 -0700 Subject: [PATCH 028/217] That sounded awkward --- param/param.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/param/param.go b/param/param.go index 0a1d8f5..685df90 100644 --- a/param/param.go +++ b/param/param.go @@ -1,5 +1,5 @@ /* -Package param deserializes parameter values into the given struct using magical +Package param deserializes parameter values into a given struct using magical reflection ponies. Inspired by gorilla/schema, but uses Rails/jQuery style param encoding instead of their weird dotted syntax. In particular, this package was written with the intent of parsing the output of jQuery.param. From 0c3f48afe0b6c61ea516858422a83b3a4152f311 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 16:26:54 -0700 Subject: [PATCH 029/217] Add an example app. I'm totally not making fun of any social networks. Promise. --- README.md | 4 ++ example/README.md | 10 +++ example/main.go | 148 ++++++++++++++++++++++++++++++++++++++++++ example/middleware.go | 47 ++++++++++++++ example/models.go | 48 ++++++++++++++ 5 files changed, 257 insertions(+) create mode 100644 example/README.md create mode 100644 example/main.go create mode 100644 example/middleware.go create mode 100644 example/models.go diff --git a/README.md b/README.md index 9a1affb..3868411 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,10 @@ func main() { } ``` +Goji also includes a [sample application][sample] in the `example` folder which +was artificially constructed to show off all of Goji's features. Check it out! + +[sample]: https://github.com/zenazn/goji/tree/master/example Features -------- diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..c3b2da0 --- /dev/null +++ b/example/README.md @@ -0,0 +1,10 @@ +Gritter +======= + +Gritter is an example application built using Goji, where people who have +nothing better to do can post short 140-character "greets." + +A good place to start is with `main.go`, which contains a well-commented +walthrough of Goji's features. Gritter uses a couple custom middlwares, which +have been arbitrarily placed in `middleware.go`. Finally some uninteresting +"database models" live in `models.go`. diff --git a/example/main.go b/example/main.go new file mode 100644 index 0000000..a0dcc69 --- /dev/null +++ b/example/main.go @@ -0,0 +1,148 @@ +// Package example is a sample application built with Goji. Its goal is to give +// you a taste for what Goji looks like in the real world by artificially using +// all of its features. +// +// In particular, this is a complete working site for gritter.com, a site where +// users can post 140-character "greets". Any resemblance to real websites, +// alive or dead, is purely coincidental. +package main + +import ( + "fmt" + "io" + "net/http" + "regexp" + "strconv" + + "github.com/zenazn/goji" + "github.com/zenazn/goji/param" + "github.com/zenazn/goji/web" +) + +// Note: the code below cuts a lot of corners to make the example app simple. + +func main() { + // Add routes to the global handler + goji.Get("/", Root) + // Fully backwards compatible with net/http's Handlers + goji.Get("/greets", http.RedirectHandler("/", 301)) + // Use your favorite HTTP verbs + goji.Post("/greets", NewGreet) + // Use Sinatra-style patterns in your URLs + goji.Get("/users/:name", GetUser) + // Goji also supports regular expressions with named capture groups. + goji.Get(regexp.MustCompile(`^/greets/(?P\d+)$`), GetGreet) + + // Middleware can be used to inject behavior into your app. The + // middleware for this application are defined in middleware.go, but you + // can put them wherever you like. + goji.Use("PlainText", PlainText) + + // Sub-routes can be used to set custom middleware on sub-applications. + // Goji's interfaces are completely composable. + admin := web.New() + goji.Sub("/admin", admin) + + // Insert the super-secure middleware into the stack above the built-in + // AutomaticOptions middleware. + admin.Insert("SuperSecure", SuperSecure, "AutomaticOptions") + + // Set up admin routes. Note that sub-routes do *not* mutate the path in + // any way, so we need to supply full ("/admin" prefixed) paths. + admin.Get("/admin", AdminRoot) + admin.Get("/admin/finances", AdminFinances) + + // Use a custom 404 handler + goji.NotFound(NotFound) + + // Call Serve() at the bottom of your main() function, and it'll take + // care of everything else for you, including binding to a socket (with + // automatic support for systemd and Einhorn) and supporting graceful + // shutdown on SIGINT. Serve() is appropriate for both development and + // production. + goji.Serve() +} + +// Root route (GET "/"). Print a list of greets. +func Root(w http.ResponseWriter, r *http.Request) { + // In the real world you'd probably use a template or something. + io.WriteString(w, "Gritter\n======\n\n") + for i := len(Greets) - 1; i >= 0; i-- { + Greets[i].Write(w) + } +} + +// Create a new greet (POST "/greets"). Creates a greet and redirects you to the +// created greet. +// +// To post a new greet, try this at a shell: +// $ now=$(date +'%Y-%m-%mT%H:%M:%SZ') +// $ curl -i -d "user=carl&message=Hello+World&time=$now" localhost:8000/greets +func NewGreet(w http.ResponseWriter, r *http.Request) { + var greet Greet + + // Parse the POST body into the Greet struct. The format is the same as + // is emitted by (e.g.) jQuery.params. + r.ParseForm() + err := param.Parse(r.Form, &greet) + + if err != nil || len(greet.Message) > 140 { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // We make no effort to prevent races against other insertions. + Greets = append(Greets, greet) + url := fmt.Sprintf("/greets/%d", len(Greets)-1) + http.Redirect(w, r, url, http.StatusCreated) +} + +// Get a given user and her greets (GET "/user/:name") +func GetUser(c web.C, w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "Gritter\n======\n\n") + handle := c.UrlParams["name"] + user, ok := Users[handle] + if !ok { + http.Error(w, http.StatusText(404), 404) + return + } + + user.Write(w, handle) + + io.WriteString(w, "\nGreets:\n") + for i := len(Greets) - 1; i >= 0; i-- { + if Greets[i].User == handle { + Greets[i].Write(w) + } + } +} + +// Get a particular greet by ID (GET "/greet/\d+"). Does no bounds checking, so +// will probably panic. +func GetGreet(c web.C, w http.ResponseWriter, r *http.Request) { + id, err := strconv.Atoi(c.UrlParams["id"]) + if err != nil { + http.Error(w, http.StatusText(404), 404) + return + } + // This will panic if id is too big. Try it out! + greet := Greets[id] + + io.WriteString(w, "Gritter\n======\n\n") + greet.Write(w) +} + +// Admin root (GET "/admin/root"). Much secret. Very administrate. Wow. +func AdminRoot(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "Gritter\n======\n\nSuper secret admin page!\n") +} + +// How are we doing? (GET "/admin/finances") +func AdminFinances(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "Gritter\n======\n\nWe're broke! :(\n") +} + +// 404 handler. +func NotFound(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Umm... have you tried turning it off and on again?", 404) +} diff --git a/example/middleware.go b/example/middleware.go new file mode 100644 index 0000000..1b9a0ab --- /dev/null +++ b/example/middleware.go @@ -0,0 +1,47 @@ +package main + +import ( + "encoding/base64" + "net/http" + "strings" + + "github.com/zenazn/goji/web" +) + +// Middleware to render responses as plain text. +func PlainText(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} + +// Nobody will ever guess this! +const Password = "admin:admin" + +// HTTP Basic Auth middleware for super-secret admin page. Shhhh! +func SuperSecure(c *web.C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + if !strings.HasPrefix(auth, "Basic ") { + pleaseAuth(w) + return + } + + password, err := base64.StdEncoding.DecodeString(auth[6:]) + if err != nil || string(password) != Password { + pleaseAuth(w) + return + } + + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} + +func pleaseAuth(w http.ResponseWriter) { + w.Header().Set("WWW-Authenticate", `Basic realm="Gritter"`) + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte("Go away!\n")) +} diff --git a/example/models.go b/example/models.go new file mode 100644 index 0000000..ef9453c --- /dev/null +++ b/example/models.go @@ -0,0 +1,48 @@ +package main + +import ( + "fmt" + "io" + "time" +) + +// A greet is a 140-character micro-blogpost that has no resemblance whatsoever +// to the noise a bird makes. +type Greet struct { + User string `param:"user"` + Message string `param:"message"` + Time time.Time `param:"time"` +} + +// Store all our greets in a big list in memory, because, let's be honest, who's +// actually going to use a service that only allows you to post 140-character +// messages? +var Greets = []Greet{ + {"carl", "Welcome to Gritter!", time.Now()}, + {"alice", "Wanna know a secret?", time.Now()}, + {"bob", "Okay!", time.Now()}, + {"eve", "I'm listening...", time.Now()}, +} + +// Write out a representation of the greet +func (g Greet) Write(w io.Writer) { + fmt.Fprintf(w, "%s\n@%s at %s\n---\n", g.Message, g.User, + g.Time.Format(time.UnixDate)) +} + +// A user +type User struct { + Name, Bio string +} + +// All the users we know about! There aren't very many... +var Users = map[string]User{ + "alice": {"Alice in Wonderland", "Eating mushrooms"}, + "bob": {"Bob the Builder", "Making children dumber"}, + "carl": {"Carl Jackson", "Duct tape aficionado"}, +} + +// Write out the user +func (u User) Write(w io.Writer, handle string) { + fmt.Fprintf(w, "%s (@%s)\n%s\n", u.Name, handle, u.Bio) +} From 282f11b5b38929f88733ddd8770fdb54690317d7 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 16:36:12 -0700 Subject: [PATCH 030/217] Tpyo --- example/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/README.md b/example/README.md index c3b2da0..acb84d6 100644 --- a/example/README.md +++ b/example/README.md @@ -5,6 +5,6 @@ Gritter is an example application built using Goji, where people who have nothing better to do can post short 140-character "greets." A good place to start is with `main.go`, which contains a well-commented -walthrough of Goji's features. Gritter uses a couple custom middlwares, which +walthrough of Goji's features. Gritter uses a couple custom middlewares, which have been arbitrarily placed in `middleware.go`. Finally some uninteresting "database models" live in `models.go`. From 4a8029a1b49c2609873bcfe791f98a6b709992ac Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 18:16:39 -0700 Subject: [PATCH 031/217] Write more words --- README.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/README.md b/README.md index 3868411..0387302 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Goji is a minimalistic web framework inspired by Sinatra. [Godoc][doc]. [doc]: http://godoc.org/github.com/zenazn/goji + Example ------- @@ -34,6 +35,7 @@ was artificially constructed to show off all of Goji's features. Check it out! [sample]: https://github.com/zenazn/goji/tree/master/example + Features -------- @@ -51,3 +53,75 @@ Features [graceful]: http://godoc.org/github.com/zenazn/goji/graceful [param]: http://godoc.org/github.com/zenazn/goji/param + +Is it any good? +--------------- + +Maybe! + +There are [plenty][revel] of [other][gorilla] [good][pat] [Go][martini] +[web][gocraft] [frameworks][tiger] out there. Goji is by no means especially +novel, nor is it uniquely good. The primary difference between Goji and other +frameworks--and the primary reason I think Goji is any good--is its philosophy: + +Goji first of all attempts to be simple. It is of the Sinatra school of web +framework design, and not the Rails one. If you want me to tell you what +directory you should put your models in, or if you want built-in flash sessions, +you won't have a good time with Goji. + +Secondly, Goji attempts to be composable. It is fully composable with net/http, +and can be used as a `http.Handler`, or can serve arbitrary `http.Handler`s. At +least a few HTTP frameworks share this property, and is not particularly novel. +The more interesting property in my mind is that Goji is fully composable with +itself: it defines an interface (`web.Handler`) which is both fully compatible +with `http.Handler` and allows Goji to perform a "protocol upgrade" of sorts +when it detects that it is talking to itself (or another `web.Handler` +compatible component). `web.Handler` is at the core of Goji's interfaces and is +what allows it to share request contexts across unrelated objects. + +Third, Goji is not magic. One of my favorite existing frameworks is +[Martini][martini], but I rejected it in favor of building Goji because I +thought it was too magical. Goji's web package does not use reflection at all, +which is not in itself a sign of API quality, but to me at least seems to +suggest it. + +Finally, Goji gives you enough rope to hang yourself with. One of my other +favorite libraries, [pat][pat], implements Sinatra-like routing in a +particularly elegant way, but because of its reliance on net/http's interfaces, +doesn't allow programmers to thread their own state through the request handling +process. Implementing arbitrary context objects was one of the primary +motivations behind abandoning pat to write Goji. + +[revel]: http://revel.github.io/ +[gorilla]: http://www.gorillatoolkit.org/ +[pat]: https://github.com/bmizerany/pat +[martini]: http://martini.codegangsta.io/ +[gocraft]: https://github.com/gocraft/web +[tiger]: https://github.com/rcrowley/go-tigertonic + + +Is it fast? +----------- + +It's not bad: in very informal tests it performed roughly in the middle of the +pack of [one set of benchmarks][bench]. For almost all applications this means +that it's fast enough that it doesn't matter. + +I have very little interest in boosting Goji's router's benchmark scores. There +is an obvious solution here--radix trees--and maybe if I get bored I'll +implement one for Goji, but I think the API guarantees and conceptual simplicity +Goji provides are more important (all routes are attempted, one after another, +until a matching route is found). Even if I choose to optimize Goji's router, +Goji's routing semantics will not change. + +Plus, Goji provides users with the ability to create their own radix trees: by +using sub-routes you create a tree of routers and match routes in more or less +the same way as a radix tree would. But, again, the real win here in my mind +isn't the performance, but the separation of concerns you get from having your +`/admin` routes and your `/profile` routes far, far away from each other. + +Goji's performance isn't all about the router though, it's also about allowing +net/http to perform its built-in optimizations. Perhaps uniquely in the Go web +framework ecosystem, Goji supports net/http's transparent `sendfile(2)` support. + +[bench]: https://github.com/cypriss/golang-mux-benchmark/ From 59fffcee72e81dcf07826ab6e17fb045daafcd23 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 18:23:50 -0700 Subject: [PATCH 032/217] License and contributing section --- LICENSE | 20 ++++++++++++++++++++ README.md | 13 +++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..777b8f4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2014 Carl Jackson (carl@avtok.com) + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index 0387302..8c261d2 100644 --- a/README.md +++ b/README.md @@ -125,3 +125,16 @@ net/http to perform its built-in optimizations. Perhaps uniquely in the Go web framework ecosystem, Goji supports net/http's transparent `sendfile(2)` support. [bench]: https://github.com/cypriss/golang-mux-benchmark/ + + +Contributing +------------ + +Please do! I love pull requests, and I love pull requests that include tests +even more. Goji's core packages have pretty good code coverage (yay code +coverage gamification!), and if you have the time to write tests I'd like to +keep it that way. + +In addition to contributing code, I'd love to know what you think about Goji. +Please open an issue or send me an email with your thoughts; it'd mean a lot to +me. From b8a86e94617f2e424e07fbae4913a934e51e4361 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 20:50:15 -0700 Subject: [PATCH 033/217] More whimsy --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8c261d2..0120cbc 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ Features * [Graceful shutdown][graceful], and zero-downtime graceful reload when combined with Einhorn. * Ruby on Rails / jQuery style [parameter parsing][param] +* High in antioxidants [einhorn]: https://github.com/stripe/einhorn [bind]: http://godoc.org/github.com/zenazn/goji/bind From bc7d07dec0efde02f6287b6af2621085d95b155b Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 22:49:49 -0700 Subject: [PATCH 034/217] Fix example app I guess I forgot to test it after changing it to an Insert() --- example/main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/example/main.go b/example/main.go index a0dcc69..71b18f5 100644 --- a/example/main.go +++ b/example/main.go @@ -42,10 +42,7 @@ func main() { // Goji's interfaces are completely composable. admin := web.New() goji.Sub("/admin", admin) - - // Insert the super-secure middleware into the stack above the built-in - // AutomaticOptions middleware. - admin.Insert("SuperSecure", SuperSecure, "AutomaticOptions") + admin.Use("SuperSecure", SuperSecure) // Set up admin routes. Note that sub-routes do *not* mutate the path in // any way, so we need to supply full ("/admin" prefixed) paths. From d235950870430cd82ad898ba50fe53ca11ab9b9b Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Mar 2014 22:53:23 -0700 Subject: [PATCH 035/217] More copy editing --- example/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/main.go b/example/main.go index 71b18f5..54baf2a 100644 --- a/example/main.go +++ b/example/main.go @@ -79,7 +79,7 @@ func NewGreet(w http.ResponseWriter, r *http.Request) { var greet Greet // Parse the POST body into the Greet struct. The format is the same as - // is emitted by (e.g.) jQuery.params. + // is emitted by (e.g.) jQuery.param. r.ParseForm() err := param.Parse(r.Form, &greet) From 83f7dab44e4dbf4b6b17adda1e1c3ec830d6cf6a Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 30 Mar 2014 16:10:35 -0700 Subject: [PATCH 036/217] The example app is a command --- example/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/main.go b/example/main.go index 54baf2a..31329b9 100644 --- a/example/main.go +++ b/example/main.go @@ -1,4 +1,4 @@ -// Package example is a sample application built with Goji. Its goal is to give +// Command example is a sample application built with Goji. Its goal is to give // you a taste for what Goji looks like in the real world by artificially using // all of its features. // From b29762524d24ccf7e461899b9a6eb370fde5f0ad Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 30 Mar 2014 16:32:33 -0700 Subject: [PATCH 037/217] Fix router tests These were introduced by breaking changes in the following commits: - aca58b22cfe17db7c5fc9a94c1ab647c6783cd08 (ValidMethodsKey) - 386323a6edcd74962da674f36df8489f5b1fbe4b (HTTP methods are case sensitive) --- web/router_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/web/router_test.go b/web/router_test.go index 44331ae..84c0f62 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -34,8 +34,8 @@ func TestMethods(t *testing.T) { rt.Connect("/", chHandler(ch, "CONNECT")) rt.Delete("/", chHandler(ch, "DELETE")) - rt.Get("/", chHandler(ch, "GET")) rt.Head("/", chHandler(ch, "HEAD")) + rt.Get("/", chHandler(ch, "GET")) rt.Options("/", chHandler(ch, "OPTIONS")) rt.Patch("/", chHandler(ch, "PATCH")) rt.Post("/", chHandler(ch, "POST")) @@ -50,7 +50,7 @@ func TestMethods(t *testing.T) { select { case val := <-ch: if val != method { - t.Error("Got %q, expected %q", val, method) + t.Errorf("Got %q, expected %q", val, method) } case <-time.After(5 * time.Millisecond): t.Errorf("Timeout waiting for method %q", method) @@ -117,7 +117,7 @@ func TestHandlerTypes(t *testing.T) { rt.Get("/e", testHandler(ch)) for route, response := range testHandlerTable { - r, _ := http.NewRequest("gEt", route, nil) + r, _ := http.NewRequest("GET", route, nil) w := httptest.NewRecorder() rt.route(C{}, w, r) select { @@ -147,7 +147,7 @@ func TestNotFound(t *testing.T) { http.Error(w, "I'm a teapot!", http.StatusTeapot) }) - r, _ = http.NewRequest("post", "/", nil) + r, _ = http.NewRequest("POST", "/", nil) w = httptest.NewRecorder() rt.route(C{}, w, r) if w.Code != http.StatusTeapot { @@ -178,9 +178,9 @@ func TestSub(t *testing.T) { } var validMethodsTable = map[string][]string{ - "/hello/carl": {"DELETE", "GET", "PATCH", "POST", "PUT"}, + "/hello/carl": {"DELETE", "GET", "HEAD", "PATCH", "POST", "PUT"}, "/hello/bob": {"DELETE", "GET", "HEAD", "PATCH", "PUT"}, - "/hola/carl": {"DELETE", "GET", "PUT"}, + "/hola/carl": {"DELETE", "GET", "HEAD", "PUT"}, "/hola/bob": {"DELETE"}, "/does/not/compute": {}, } @@ -195,7 +195,7 @@ func TestValidMethods(t *testing.T) { ch <- []string{} return } - methods, ok := c.Env[validMethods] + methods, ok := c.Env[ValidMethodsKey] if !ok { ch <- []string{} return From 35af451420a3feb90c49ebabd1bf1f1775d02755 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 30 Mar 2014 16:34:09 -0700 Subject: [PATCH 038/217] Replace Sub() with trailing * in patterns This change replaces a bit of API surface area (the Sub() method on Muxes) with a slightly more expressive pattern syntax. I'm mostly doing this because it seems cleaner: the "*" gets to take on a meaning very similar to what it means in Sinatra (without growing regexp-like middle-of-a-path globbing, which sounds terrifying and not particularly useful), and we get to nuke a useless function from the API. --- default.go | 6 ------ example/main.go | 4 +++- web/mux.go | 17 +++++++++++++---- web/pattern.go | 9 ++++++++- web/pattern_test.go | 25 +++++++++++++++++++------ web/router.go | 44 +++++++++++++++----------------------------- web/router_test.go | 4 ++-- 7 files changed, 60 insertions(+), 49 deletions(-) diff --git a/default.go b/default.go index 6ec68ed..70b8ed3 100644 --- a/default.go +++ b/default.go @@ -101,12 +101,6 @@ func Trace(pattern interface{}, handler interface{}) { DefaultMux.Trace(pattern, handler) } -// Add a sub-route to the default Mux. See the documentation for web.Mux.Sub -// for more information. -func Sub(pattern string, handler interface{}) { - DefaultMux.Sub(pattern, handler) -} - // Set the NotFound handler for the default Mux. See the documentation for // web.Mux.NotFound for more information. func NotFound(handler interface{}) { diff --git a/example/main.go b/example/main.go index 31329b9..3499aa2 100644 --- a/example/main.go +++ b/example/main.go @@ -38,10 +38,12 @@ func main() { // can put them wherever you like. goji.Use("PlainText", PlainText) + // If the last character of a pattern is an asterisk, the path is + // treated as a prefix, and can be used to implement sub-routes. // Sub-routes can be used to set custom middleware on sub-applications. // Goji's interfaces are completely composable. admin := web.New() - goji.Sub("/admin", admin) + goji.Handle("/admin/*", admin) admin.Use("SuperSecure", SuperSecure) // Set up admin routes. Note that sub-routes do *not* mutate the path in diff --git a/web/mux.go b/web/mux.go index 00b5d68..33f92c5 100644 --- a/web/mux.go +++ b/web/mux.go @@ -30,11 +30,20 @@ following types: - func(c *web.C, http.Handler) http.Handler All of the route-adding functions on Mux take two untyped parameters: pattern and handler. Pattern must be one of the following types: - - string (interpreted as a Sinatra pattern) + - string. It will be interpreted as a Sinatra-like pattern. In + particular, the following syntax is recognized: + - a path segment starting with with a colon will match any + string placed at that position. e.g., "/:name" will match + "/carl", binding "name" to "carl". + - a pattern ending with an asterisk will match any prefix of + that route. For instance, "/admin/*" will match "/admin/" and + "/admin/secret/lair". This is similar to Sinatra's wildcard, + but may only appear at the very end of the string and is + therefore significantly less powerful. - regexp.Regexp. The library assumes that it is a Perl-style regexp that - is anchored on the left (i.e., the beginning of the string). If your - regexp is not anchored on the left, a hopefully-identical left-anchored - regexp will be created and used instead. + is anchored on the left (i.e., the beginning of the string). If your + regexp is not anchored on the left, a hopefully-identical + left-anchored regexp will be created and used instead. - web.Pattern Handler must be one of the following types: - http.Handler diff --git a/web/pattern.go b/web/pattern.go index 4916827..401363d 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -200,7 +200,14 @@ func (s stringPattern) String() string { var patternRe = regexp.MustCompile(`/:([^/]+)`) -func parseStringPattern(s string, isPrefix bool) stringPattern { +func parseStringPattern(s string) stringPattern { + var isPrefix bool + // Routes that end in an asterisk ("*") are prefix routes + if len(s) > 0 && s[len(s)-1] == '*' { + s = s[:len(s)-1] + isPrefix = true + } + matches := patternRe.FindAllStringSubmatchIndex(s, -1) pats := make([]string, len(matches)) literals := make([]string, len(matches)+1) diff --git a/web/pattern_test.go b/web/pattern_test.go index b6911d7..0377ab0 100644 --- a/web/pattern_test.go +++ b/web/pattern_test.go @@ -81,14 +81,14 @@ var patternTests = []struct { }}, // String pattern tests - {parseStringPattern("/hello", false), + {parseStringPattern("/hello"), "/hello", []patternTest{ pt("/hello", true, nil), pt("/hell", false, nil), pt("/hello/", false, nil), pt("/hello/world", false, nil), }}, - {parseStringPattern("/hello/:name", false), + {parseStringPattern("/hello/:name"), "/hello/", []patternTest{ pt("/hello/world", true, map[string]string{ "name": "world", @@ -97,7 +97,7 @@ var patternTests = []struct { pt("/hello/", false, nil), pt("/hello/my/love", false, nil), }}, - {parseStringPattern("/a/:a/b/:b", false), + {parseStringPattern("/a/:a/b/:b"), "/a/", []patternTest{ pt("/a/1/b/2", true, map[string]string{ "a": "1", @@ -108,8 +108,8 @@ var patternTests = []struct { pt("/a/1/b/2/3", false, nil), }}, - // String sub-pattern tests - {parseStringPattern("/user/:user", true), + // String prefix tests + {parseStringPattern("/user/:user*"), "/user/", []patternTest{ pt("/user/bob", true, map[string]string{ "user": "bob", @@ -120,7 +120,16 @@ var patternTests = []struct { pt("/user/", false, nil), pt("/user//", false, nil), }}, - {parseStringPattern("/user/:user/friends", true), + {parseStringPattern("/user/:user/*"), + "/user/", []patternTest{ + pt("/user/bob/friends/123", true, map[string]string{ + "user": "bob", + }), + pt("/user/bob", false, nil), + pt("/user/", false, nil), + pt("/user//", false, nil), + }}, + {parseStringPattern("/user/:user/friends*"), "/user/", []patternTest{ pt("/user/bob/friends", true, map[string]string{ "user": "bob", @@ -128,6 +137,10 @@ var patternTests = []struct { pt("/user/bob/friends/123", true, map[string]string{ "user": "bob", }), + // This is a little unfortunate + pt("/user/bob/friends123", true, map[string]string{ + "user": "bob", + }), pt("/user/bob/enemies", false, nil), }}, } diff --git a/web/router.go b/web/router.go index 770f2d9..b91dce6 100644 --- a/web/router.go +++ b/web/router.go @@ -83,14 +83,14 @@ type Pattern interface { Match(r *http.Request, c *C, dryrun bool) bool } -func parsePattern(p interface{}, isPrefix bool) Pattern { +func parsePattern(p interface{}) Pattern { switch p.(type) { case Pattern: return p.(Pattern) case *regexp.Regexp: return parseRegexpPattern(p.(*regexp.Regexp)) case string: - return parseStringPattern(p.(string), isPrefix) + return parseStringPattern(p.(string)) default: log.Fatalf("Unknown pattern type %v. Expected a web.Pattern, "+ "regexp.Regexp, or a string.", p) @@ -179,7 +179,7 @@ func (rt *router) route(c C, w http.ResponseWriter, r *http.Request) { } func (rt *router) handleUntyped(p interface{}, m method, h interface{}) { - pat := parsePattern(p, false) + pat := parsePattern(p) rt.handle(pat, m, parseHandler(h)) } @@ -201,9 +201,18 @@ func (rt *router) handle(p Pattern, m method, h Handler) { // functions here "m" instead of the standard "rt", since they will eventually // be shown on the documentation for the Mux that they are included in. -// Dispatch to the given handler when the pattern matches, regardless of HTTP -// method. See the documentation for type Mux for a description of what types -// are accepted for pattern and handler. +/* +Dispatch to the given handler when the pattern matches, regardless of HTTP +method. See the documentation for type Mux for a description of what types are +accepted for pattern and handler. + +This method is commonly used to implement sub-routing: an admin application, for +instance, can expose a single handler that is attached to the main Mux by +calling Handle("/admin*", adminHandler) or similar. Note that this function +doesn't strip this prefix from the path before forwarding it on (e.g., the +handler will see the full path, including the "/admin" part), but this +functionality can easily be performed by an extra middleware layer. +*/ func (m *router) Handle(pattern interface{}, handler interface{}) { m.handleUntyped(pattern, mALL, handler) } @@ -276,29 +285,6 @@ func (m *router) Trace(pattern interface{}, handler interface{}) { m.handleUntyped(pattern, mTRACE, handler) } -/* -Dispatch to the given handler when the given (Sinatra-like) pattern matches a -prefix of the path. This function explicitly takes a string parameter since you -can implement this behavior with a regular expression using the standard -Handle() function. - -This function is probably most helpful when implementing sub-routing: an admin -application, for instance, can expose a single handler, and can be hooked up all -at once by attaching a sub-route at "/admin". - -Notably, this function does not strip the matched prefix from downstream -handlers, so in the above example the handler would recieve requests with path -"/admin/foo/bar", for example, instead of just "/foo/bar". Luckily, this is a -problem easily surmountable by middleware. - -See the documentation for type Mux for a description of what types are accepted -for handler. -*/ -func (m *router) Sub(pattern string, handler interface{}) { - pat := parsePattern(pattern, true) - m.handle(pat, mALL, parseHandler(handler)) -} - // Set the fallback (i.e., 404) handler for this mux. See the documentation for // type Mux for a description of what types are accepted for handler. // diff --git a/web/router_test.go b/web/router_test.go index 84c0f62..5e4d7cd 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -155,12 +155,12 @@ func TestNotFound(t *testing.T) { } } -func TestSub(t *testing.T) { +func TestPrefix(t *testing.T) { t.Parallel() rt := makeRouter() ch := make(chan string, 1) - rt.Sub("/hello", func(w http.ResponseWriter, r *http.Request) { + rt.Handle("/hello*", func(w http.ResponseWriter, r *http.Request) { ch <- r.URL.Path }) From 444fd9873e534ca27dc582b72a261d8aee88ad14 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 30 Mar 2014 19:15:12 -0700 Subject: [PATCH 039/217] Deep (i.e., "real") function equality --- web/func_equal.go | 38 +++++++++++++++++++ web/func_equal_test.go | 84 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 web/func_equal.go create mode 100644 web/func_equal_test.go diff --git a/web/func_equal.go b/web/func_equal.go new file mode 100644 index 0000000..3206b04 --- /dev/null +++ b/web/func_equal.go @@ -0,0 +1,38 @@ +package web + +import ( + "reflect" +) + +func isFunc(fn interface{}) bool { + return reflect.ValueOf(fn).Kind() == reflect.Func +} + +/* +This is more than a little sketchtacular. Go's rules for function pointer +equality are pretty restrictive: nil function pointers always compare equal, and +all other pointer types never do. However, this is pretty limiting: it means +that we can't let people reference the middleware they've given us since we have +no idea which function they're referring to. + +To get better data out of Go, we sketch on the representation of interfaces. We +happen to know that interfaces are pairs of pointers: one to the real data, one +to data about the type. Therefore, two interfaces, including two function +interface{}'s, point to exactly the same objects iff their interface +representations are identical. And it turns out this is sufficient for our +purposes. + +If you're curious, you can read more about the representation of functions here: +http://golang.org/s/go11func +We're in effect comparing the pointers of the indirect layer. +*/ +func funcEqual(a, b interface{}) bool { + if !isFunc(a) || !isFunc(b) { + panic("funcEqual: type error!") + } + + av := reflect.ValueOf(&a).Elem() + bv := reflect.ValueOf(&b).Elem() + + return av.InterfaceData() == bv.InterfaceData() +} diff --git a/web/func_equal_test.go b/web/func_equal_test.go new file mode 100644 index 0000000..daf8d9a --- /dev/null +++ b/web/func_equal_test.go @@ -0,0 +1,84 @@ +package web + +import ( + "testing" +) + +// To tell you the truth, I'm not actually sure how many of these cases are +// needed. Presumably someone with more patience than I could comb through +// http://golang.org/s/go11func and figure out what all the different cases I +// ought to test are, but I think this test includes all the cases I care about +// and is at least reasonably thorough. + +func a() string { + return "A" +} +func b() string { + return "B" +} +func mkFn(s string) func() string { + return func() string { + return s + } +} + +var c = mkFn("C") +var d = mkFn("D") +var e = a +var f = c +var g = mkFn("D") + +type Type string + +func (t *Type) String() string { + return string(*t) +} + +var t1 = Type("hi") +var t2 = Type("bye") +var t1f = t1.String +var t2f = t2.String + +var funcEqualTests = []struct { + a, b func() string + result bool +}{ + {a, a, true}, + {a, b, false}, + {b, b, true}, + {a, c, false}, + {c, c, true}, + {c, d, false}, + {a, e, true}, + {a, f, false}, + {c, f, true}, + {e, f, false}, + {d, g, false}, + {t1f, t1f, true}, + {t1f, t2f, false}, +} + +func TestFuncEqual(t *testing.T) { + t.Parallel() + + for _, test := range funcEqualTests { + r := funcEqual(test.a, test.b) + if r != test.result { + t.Errorf("funcEqual(%v, %v) should have been %v", + test.a, test.b, test.result) + } + } + h := mkFn("H") + i := h + j := mkFn("H") + k := a + if !funcEqual(h, i) { + t.Errorf("h and i should have been equal") + } + if funcEqual(h, j) { + t.Errorf("h and j should not have been equal") + } + if !funcEqual(a, k) { + t.Errorf("a and k should not have been equal") + } +} From 8a7d32ac92aa05963280b6f62de8b56cd9ffde14 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 30 Mar 2014 19:36:12 -0700 Subject: [PATCH 040/217] Get rid of name parameter in middleware The "name" parameter was originally a workaround for the fact that function pointers in Go never compare equal to each other (unless they are both nil). Unfortunately, this presents a pretty terrible interface to the end programmer, since they'll probably end up stuttering something like: mux.Use("MyMiddleware", MyMiddleware) Luckily, I found a workaround for doing function pointer equality (it doesn't even require "unsafe"!), so we can get rid of the name parameter for good. --- default.go | 26 ++++++--------- example/main.go | 4 +-- web/middleware.go | 46 ++++++++++---------------- web/middleware_test.go | 75 +++++++++++++++++------------------------- 4 files changed, 60 insertions(+), 91 deletions(-) diff --git a/default.go b/default.go index 70b8ed3..a3d3911 100644 --- a/default.go +++ b/default.go @@ -11,34 +11,28 @@ var DefaultMux *web.Mux func init() { DefaultMux = web.New() - DefaultMux.Use("RequestId", middleware.RequestId) - DefaultMux.Use("Logger", middleware.Logger) - DefaultMux.Use("Recoverer", middleware.Recoverer) - DefaultMux.Use("AutomaticOptions", middleware.AutomaticOptions) + DefaultMux.Use(middleware.RequestId) + DefaultMux.Use(middleware.Logger) + DefaultMux.Use(middleware.Recoverer) + DefaultMux.Use(middleware.AutomaticOptions) } // Append the given middleware to the default Mux's middleware stack. See the // documentation for web.Mux.Use for more informatino. -func Use(name string, middleware interface{}) { - DefaultMux.Use(name, middleware) +func Use(middleware interface{}) { + DefaultMux.Use(middleware) } // Insert the given middleware into the default Mux's middleware stack. See the // documentation for web.Mux.Insert for more informatino. -func Insert(name string, middleware interface{}, before string) error { - return DefaultMux.Insert(name, middleware, before) +func Insert(middleware, before interface{}) error { + return DefaultMux.Insert(middleware, before) } // Remove the given middleware from the default Mux's middleware stack. See the // documentation for web.Mux.Abandon for more informatino. -func Abandon(name string) error { - return DefaultMux.Abandon(name) -} - -// Retrieve the list of middleware from the default Mux's middleware stack. See -// the documentation for web.Mux.Middleware() for more information. -func Middleware() []string { - return DefaultMux.Middleware() +func Abandon(middleware interface{}) error { + return DefaultMux.Abandon(middleware) } // Add a route to the default Mux. See the documentation for web.Mux for more diff --git a/example/main.go b/example/main.go index 3499aa2..076948b 100644 --- a/example/main.go +++ b/example/main.go @@ -36,7 +36,7 @@ func main() { // Middleware can be used to inject behavior into your app. The // middleware for this application are defined in middleware.go, but you // can put them wherever you like. - goji.Use("PlainText", PlainText) + goji.Use(PlainText) // If the last character of a pattern is an asterisk, the path is // treated as a prefix, and can be used to implement sub-routes. @@ -44,7 +44,7 @@ func main() { // Goji's interfaces are completely composable. admin := web.New() goji.Handle("/admin/*", admin) - admin.Use("SuperSecure", SuperSecure) + admin.Use(SuperSecure) // Set up admin routes. Note that sub-routes do *not* mutate the path in // any way, so we need to supply full ("/admin" prefixed) paths. diff --git a/web/middleware.go b/web/middleware.go index b62c67a..12c6eae 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -12,7 +12,7 @@ const mPoolSize = 32 type mLayer struct { fn func(*C, http.Handler) http.Handler - name string + orig interface{} } type mStack struct { @@ -41,9 +41,8 @@ func (s *cStack) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { s.m.ServeHTTP(w, r) } -func (m *mStack) appendLayer(name string, fn interface{}) { - var ml mLayer - ml.name = name +func (m *mStack) appendLayer(fn interface{}) { + ml := mLayer{orig: fn} switch fn.(type) { case func(http.Handler) http.Handler: unwrapped := fn.(func(http.Handler) http.Handler) @@ -60,9 +59,9 @@ func (m *mStack) appendLayer(name string, fn interface{}) { m.stack = append(m.stack, ml) } -func (m *mStack) findLayer(name string) int { +func (m *mStack) findLayer(l interface{}) int { for i, middleware := range m.stack { - if middleware.name == name { + if funcEqual(l, middleware.orig) { return i } } @@ -138,11 +137,11 @@ func (m *mStack) release(cs *cStack) { // Append the given middleware to the middleware stack. See the documentation // for type Mux for a list of valid middleware types. // -// No attempt is made to enforce the uniqueness of middleware names. -func (m *mStack) Use(name string, middleware interface{}) { +// No attempt is made to enforce the uniqueness of middlewares. +func (m *mStack) Use(middleware interface{}) { m.lock.Lock() defer m.lock.Unlock() - m.appendLayer(name, middleware) + m.appendLayer(middleware) m.invalidate() } @@ -150,9 +149,9 @@ func (m *mStack) Use(name string, middleware interface{}) { // the stack. See the documentation for type Mux for a list of valid middleware // types. Returns an error if no middleware has the name given by "before." // -// No attempt is made to enforce the uniqueness of names. If the insertion point -// is ambiguous, the first (outermost) one is chosen. -func (m *mStack) Insert(name string, middleware interface{}, before string) error { +// No attempt is made to enforce the uniqueness of middlewares. If the insertion +// point is ambiguous, the first (outermost) one is chosen. +func (m *mStack) Insert(middleware, before interface{}) error { m.lock.Lock() defer m.lock.Unlock() i := m.findLayer(before) @@ -160,7 +159,7 @@ func (m *mStack) Insert(name string, middleware interface{}, before string) erro return fmt.Errorf("web: unknown middleware %v", before) } - m.appendLayer(name, middleware) + m.appendLayer(middleware) inserted := m.stack[len(m.stack)-1] copy(m.stack[i+1:], m.stack[i:]) m.stack[i] = inserted @@ -169,17 +168,17 @@ func (m *mStack) Insert(name string, middleware interface{}, before string) erro return nil } -// Remove the given named middleware from the middleware stack. Returns an error -// if there is no middleware by that name. +// Remove the given middleware from the middleware stack. Returns an error if +// no such middleware can be found. // // If the name of the middleware to delete is ambiguous, the first (outermost) // one is chosen. -func (m *mStack) Abandon(name string) error { +func (m *mStack) Abandon(middleware interface{}) error { m.lock.Lock() defer m.lock.Unlock() - i := m.findLayer(name) + i := m.findLayer(middleware) if i < 0 { - return fmt.Errorf("web: unknown middleware %v", name) + return fmt.Errorf("web: unknown middleware %v", middleware) } copy(m.stack[i:], m.stack[i+1:]) @@ -188,14 +187,3 @@ func (m *mStack) Abandon(name string) error { m.invalidate() return nil } - -// Returns a list of middleware currently in use. -func (m *mStack) Middleware() []string { - m.lock.Lock() - defer m.lock.Unlock() - stack := make([]string, len(m.stack)) - for i, ml := range m.stack { - stack[i] = ml.name - } - return stack -} diff --git a/web/middleware_test.go b/web/middleware_test.go index 747b541..548e9b9 100644 --- a/web/middleware_test.go +++ b/web/middleware_test.go @@ -3,7 +3,6 @@ package web import ( "net/http" "net/http/httptest" - "reflect" "testing" "time" ) @@ -61,8 +60,8 @@ func TestSimple(t *testing.T) { ch := make(chan string) st := makeStack(ch) - st.Use("one", chanWare(ch, "one")) - st.Use("two", chanWare(ch, "two")) + st.Use(chanWare(ch, "one")) + st.Use(chanWare(ch, "two")) go simpleRequest(ch, st) assertOrder(t, ch, "one", "two", "router", "end") } @@ -72,10 +71,10 @@ func TestTypes(t *testing.T) { ch := make(chan string) st := makeStack(ch) - st.Use("one", func(h http.Handler) http.Handler { + st.Use(func(h http.Handler) http.Handler { return h }) - st.Use("two", func(c *C, h http.Handler) http.Handler { + st.Use(func(c *C, h http.Handler) http.Handler { return h }) } @@ -85,16 +84,16 @@ func TestAddMore(t *testing.T) { ch := make(chan string) st := makeStack(ch) - st.Use("one", chanWare(ch, "one")) + st.Use(chanWare(ch, "one")) go simpleRequest(ch, st) assertOrder(t, ch, "one", "router", "end") - st.Use("two", chanWare(ch, "two")) + st.Use(chanWare(ch, "two")) go simpleRequest(ch, st) assertOrder(t, ch, "one", "two", "router", "end") - st.Use("three", chanWare(ch, "three")) - st.Use("four", chanWare(ch, "four")) + st.Use(chanWare(ch, "three")) + st.Use(chanWare(ch, "four")) go simpleRequest(ch, st) assertOrder(t, ch, "one", "two", "three", "four", "router", "end") } @@ -104,18 +103,23 @@ func TestInsert(t *testing.T) { ch := make(chan string) st := makeStack(ch) - st.Use("one", chanWare(ch, "one")) - st.Use("two", chanWare(ch, "two")) + one := chanWare(ch, "one") + two := chanWare(ch, "two") + st.Use(one) + st.Use(two) go simpleRequest(ch, st) assertOrder(t, ch, "one", "two", "router", "end") - err := st.Insert("sloth", chanWare(ch, "sloth"), "squirrel") + err := st.Insert(chanWare(ch, "sloth"), chanWare(ch, "squirrel")) if err == nil { t.Error("Expected error when referencing unknown middleware") } - st.Insert("middle", chanWare(ch, "middle"), "two") - st.Insert("start", chanWare(ch, "start"), "one") + st.Insert(chanWare(ch, "middle"), two) + err = st.Insert(chanWare(ch, "start"), one) + if err != nil { + t.Fatal(err) + } go simpleRequest(ch, st) assertOrder(t, ch, "start", "one", "middle", "two", "router", "end") } @@ -125,51 +129,34 @@ func TestAbandon(t *testing.T) { ch := make(chan string) st := makeStack(ch) - st.Use("one", chanWare(ch, "one")) - st.Use("two", chanWare(ch, "two")) - st.Use("three", chanWare(ch, "three")) + one := chanWare(ch, "one") + two := chanWare(ch, "two") + three := chanWare(ch, "three") + st.Use(one) + st.Use(two) + st.Use(three) go simpleRequest(ch, st) assertOrder(t, ch, "one", "two", "three", "router", "end") - st.Abandon("two") + st.Abandon(two) go simpleRequest(ch, st) assertOrder(t, ch, "one", "three", "router", "end") - err := st.Abandon("panda") + err := st.Abandon(chanWare(ch, "panda")) if err == nil { t.Error("Expected error when deleting unknown middleware") } - st.Abandon("one") - st.Abandon("three") + st.Abandon(one) + st.Abandon(three) go simpleRequest(ch, st) assertOrder(t, ch, "router", "end") - st.Use("one", chanWare(ch, "one")) + st.Use(one) go simpleRequest(ch, st) assertOrder(t, ch, "one", "router", "end") } -func TestMiddlewareList(t *testing.T) { - t.Parallel() - - ch := make(chan string) - st := makeStack(ch) - st.Use("one", chanWare(ch, "one")) - st.Use("two", chanWare(ch, "two")) - st.Insert("mid", chanWare(ch, "mid"), "two") - st.Insert("before", chanWare(ch, "before"), "mid") - st.Abandon("one") - - m := st.Middleware() - if !reflect.DeepEqual(m, []string{"before", "mid", "two"}) { - t.Error("Middleware list was not as expected") - } - - go simpleRequest(ch, st) - assertOrder(t, ch, "before", "mid", "two", "router", "end") -} - // This is a pretty sketchtacular test func TestCaching(t *testing.T) { ch := make(chan string) @@ -225,7 +212,7 @@ func TestContext(t *testing.T) { pool: make(chan *cStack, mPoolSize), router: HandlerFunc(router), } - st.Use("one", func(c *C, h http.Handler) http.Handler { + st.Use(func(c *C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { if c.Env != nil || c.UrlParams != nil { t.Error("Expected a clean context") @@ -238,7 +225,7 @@ func TestContext(t *testing.T) { return http.HandlerFunc(fn) }) - st.Use("two", func(c *C, h http.Handler) http.Handler { + st.Use(func(c *C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { if c.Env == nil { t.Error("Expected env from last middleware") From 4baae23a99184796f198d81fe31c1074566c1fac Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 30 Mar 2014 19:42:55 -0700 Subject: [PATCH 041/217] Fix some typos Wonder how long those have been there. --- default.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/default.go b/default.go index a3d3911..fbfae9e 100644 --- a/default.go +++ b/default.go @@ -18,19 +18,19 @@ func init() { } // Append the given middleware to the default Mux's middleware stack. See the -// documentation for web.Mux.Use for more informatino. +// documentation for web.Mux.Use for more information. func Use(middleware interface{}) { DefaultMux.Use(middleware) } // Insert the given middleware into the default Mux's middleware stack. See the -// documentation for web.Mux.Insert for more informatino. +// documentation for web.Mux.Insert for more information. func Insert(middleware, before interface{}) error { return DefaultMux.Insert(middleware, before) } // Remove the given middleware from the default Mux's middleware stack. See the -// documentation for web.Mux.Abandon for more informatino. +// documentation for web.Mux.Abandon for more information. func Abandon(middleware interface{}) error { return DefaultMux.Abandon(middleware) } From 05c2ca7e533b3b3a5973d041fd32cdfe85f7b25a Mon Sep 17 00:00:00 2001 From: Coda Hale Date: Sat, 12 Apr 2014 11:18:17 -0700 Subject: [PATCH 042/217] Make graceful.Server a package-local type. --- graceful/graceful.go | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/graceful/graceful.go b/graceful/graceful.go index 0d51726..c6dc314 100644 --- a/graceful/graceful.go +++ b/graceful/graceful.go @@ -25,13 +25,6 @@ import ( "time" ) -// Exactly like net/http's Server. In fact, it *is* a net/http Server, just with -// different method implementations -type Server http.Server - -// About 200 years, also known as "forever" -const forever time.Duration = 200 * 365 * 24 * time.Hour - /* You might notice that these methods look awfully similar to the methods of the same name from the go standard library--that's because they were stolen from @@ -44,8 +37,9 @@ Since I couldn't come up with a better idea, I just copy-and-pasted both ListenAndServe and ListenAndServeTLS here more-or-less verbatim. "Oh well!" */ -// Behaves exactly like the net/http function of the same name. -func (srv *Server) Serve(l net.Listener) (err error) { +type server http.Server + +func (srv *server) Serve(l net.Listener) (err error) { go func() { <-kill l.Close() @@ -74,8 +68,10 @@ func (srv *Server) Serve(l net.Listener) (err error) { } } -// Behaves exactly like the net/http function of the same name. -func (srv *Server) ListenAndServe() error { +// About 200 years, also known as "forever" +const forever time.Duration = 200 * 365 * 24 * time.Hour + +func (srv *server) ListenAndServe() error { addr := srv.Addr if addr == "" { addr = ":http" @@ -87,8 +83,7 @@ func (srv *Server) ListenAndServe() error { return srv.Serve(l) } -// Behaves exactly like the net/http function of the same name. -func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { +func (srv *server) ListenAndServeTLS(certFile, keyFile string) error { addr := srv.Addr if addr == "" { addr = ":https" @@ -117,20 +112,20 @@ func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { return srv.Serve(tlsListener) } -// Behaves exactly like the net/http function of the same name. +// ListenAndServe behaves exactly like the net/http function of the same name. func ListenAndServe(addr string, handler http.Handler) error { - server := &Server{Addr: addr, Handler: handler} + server := &server{Addr: addr, Handler: handler} return server.ListenAndServe() } -// Behaves exactly like the net/http function of the same name. +// ListenAndServeTLS behaves exactly like the net/http function of the same name. func ListenAndServeTLS(addr, certfile, keyfile string, handler http.Handler) error { - server := &Server{Addr: addr, Handler: handler} + server := &server{Addr: addr, Handler: handler} return server.ListenAndServeTLS(certfile, keyfile) } -// Behaves exactly like the net/http function of the same name. +// Serve behaves exactly like the net/http function of the same name. func Serve(l net.Listener, handler http.Handler) error { - server := &Server{Handler: handler} + server := &server{Handler: handler} return server.Serve(l) } From 62d9a012fae34d9b7bc4fae91c8cffc68af59287 Mon Sep 17 00:00:00 2001 From: Coda Hale Date: Sat, 12 Apr 2014 11:33:19 -0700 Subject: [PATCH 043/217] Ensure all error messages are not capitalized. --- bind/bind.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bind/bind.go b/bind/bind.go index 5541ec9..4d7cdb3 100644 --- a/bind/bind.go +++ b/bind/bind.go @@ -67,7 +67,7 @@ func listenTo(bind string) (net.Listener, error) { } else if strings.HasPrefix(bind, "fd@") { fd, err := strconv.Atoi(bind[3:]) if err != nil { - return nil, fmt.Errorf("Error while parsing fd %v: %v", + return nil, fmt.Errorf("error while parsing fd %v: %v", bind, err) } f := os.NewFile(uintptr(fd), bind) @@ -76,16 +76,17 @@ func listenTo(bind string) (net.Listener, error) { fd, err := strconv.Atoi(bind[8:]) if err != nil { return nil, fmt.Errorf( - "Error while parsing einhorn %v: %v", bind, err) + "error while parsing einhorn %v: %v", bind, err) } return einhornBind(fd) } - return nil, fmt.Errorf("Error while parsing bind arg %v", bind) + return nil, fmt.Errorf("error while parsing bind arg %v", bind) } -// Parse and bind to the specified address. If Socket encounters an error while -// parsing or binding to the given socket it will exit by calling log.Fatal. +// Socket parses and binds to the specified address. If Socket encounters an +// error while parsing or binding to the given socket it will exit by calling +// log.Fatal. func Socket(bind string) net.Listener { l, err := listenTo(bind) if err != nil { From c912dac2b6ea206027f0eb4f8048fe418affc8c7 Mon Sep 17 00:00:00 2001 From: Coda Hale Date: Sat, 12 Apr 2014 11:35:51 -0700 Subject: [PATCH 044/217] Included documented names in documentation. --- bind/bind.go | 13 ++++++----- default.go | 48 +++++++++++++++++++-------------------- example/main.go | 17 +++++++------- example/middleware.go | 4 ++-- example/models.go | 5 ++-- goji.go | 2 +- graceful/middleware.go | 7 +++--- graceful/net.go | 12 +++++----- graceful/signal.go | 37 +++++++++++++++--------------- web/middleware/options.go | 4 ++-- web/mux.go | 8 +++---- web/router.go | 2 +- web/web.go | 15 ++++++------ 13 files changed, 90 insertions(+), 84 deletions(-) diff --git a/bind/bind.go b/bind/bind.go index 4d7cdb3..60aecc4 100644 --- a/bind/bind.go +++ b/bind/bind.go @@ -95,9 +95,9 @@ func Socket(bind string) net.Listener { return l } -// Parse and bind to the default socket as given to us by the flag module. If -// there was an error parsing or binding to that socket, Default will exit by -// calling `log.Fatal`. +// Default parses and binds to the default socket as given to us by the flag +// module. If there was an error parsing or binding to that socket, Default will +// exit by calling `log.Fatal`. func Default() net.Listener { return Socket(bind) } @@ -106,9 +106,10 @@ func Default() net.Listener { // as well be safe against it... var ready sync.Once -// Notify the environment (for now, just Einhorn) that the process is ready to -// receive traffic. Should be called at the last possible moment to maximize the -// chances that a faulty process exits before signaling that it's ready. +// Ready notifies the environment (for now, just Einhorn) that the process is +// ready to receive traffic. Should be called at the last possible moment to +// maximize the chances that a faulty process exits before signaling that it's +// ready. func Ready() { ready.Do(func() { einhornAck() diff --git a/default.go b/default.go index fbfae9e..5d78ca8 100644 --- a/default.go +++ b/default.go @@ -17,8 +17,8 @@ func init() { DefaultMux.Use(middleware.AutomaticOptions) } -// Append the given middleware to the default Mux's middleware stack. See the -// documentation for web.Mux.Use for more information. +// Use appends the given middleware to the default Mux's middleware stack. See +// the documentation for web.Mux.Use for more information. func Use(middleware interface{}) { DefaultMux.Use(middleware) } @@ -29,74 +29,74 @@ func Insert(middleware, before interface{}) error { return DefaultMux.Insert(middleware, before) } -// Remove the given middleware from the default Mux's middleware stack. See the -// documentation for web.Mux.Abandon for more information. +// Abandon removes the given middleware from the default Mux's middleware stack. +// See the documentation for web.Mux.Abandon for more information. func Abandon(middleware interface{}) error { return DefaultMux.Abandon(middleware) } -// Add a route to the default Mux. See the documentation for web.Mux for more -// information about what types this function accepts. +// Handle adds a route to the default Mux. See the documentation for web.Mux for +// more information about what types this function accepts. func Handle(pattern interface{}, handler interface{}) { DefaultMux.Handle(pattern, handler) } -// Add a CONNECT route to the default Mux. See the documentation for web.Mux for -// more information about what types this function accepts. +// Connect adds a CONNECT route to the default Mux. See the documentation for +// web.Mux for more information about what types this function accepts. func Connect(pattern interface{}, handler interface{}) { DefaultMux.Connect(pattern, handler) } -// Add a DELETE route to the default Mux. See the documentation for web.Mux for -// more information about what types this function accepts. +// Delete adds a DELETE route to the default Mux. See the documentation for +// web.Mux for more information about what types this function accepts. func Delete(pattern interface{}, handler interface{}) { DefaultMux.Delete(pattern, handler) } -// Add a GET route to the default Mux. See the documentation for web.Mux for +// Get adds a GET route to the default Mux. See the documentation for web.Mux for // more information about what types this function accepts. func Get(pattern interface{}, handler interface{}) { DefaultMux.Get(pattern, handler) } -// Add a HEAD route to the default Mux. See the documentation for web.Mux for -// more information about what types this function accepts. +// Head adds a HEAD route to the default Mux. See the documentation for web.Mux +// for more information about what types this function accepts. func Head(pattern interface{}, handler interface{}) { DefaultMux.Head(pattern, handler) } -// Add a OPTIONS route to the default Mux. See the documentation for web.Mux for -// more information about what types this function accepts. +// Options adds a OPTIONS route to the default Mux. See the documentation for +// web.Mux for more information about what types this function accepts. func Options(pattern interface{}, handler interface{}) { DefaultMux.Options(pattern, handler) } -// Add a PATCH route to the default Mux. See the documentation for web.Mux for -// more information about what types this function accepts. +// Patch adds a PATCH route to the default Mux. See the documentation for web.Mux +// for more information about what types this function accepts. func Patch(pattern interface{}, handler interface{}) { DefaultMux.Patch(pattern, handler) } -// Add a POST route to the default Mux. See the documentation for web.Mux for -// more information about what types this function accepts. +// Post adds a POST route to the default Mux. See the documentation for web.Mux +// for more information about what types this function accepts. func Post(pattern interface{}, handler interface{}) { DefaultMux.Post(pattern, handler) } -// Add a PUT route to the default Mux. See the documentation for web.Mux for +// Put adds a PUT route to the default Mux. See the documentation for web.Mux for // more information about what types this function accepts. func Put(pattern interface{}, handler interface{}) { DefaultMux.Put(pattern, handler) } -// Add a TRACE route to the default Mux. See the documentation for web.Mux for -// more information about what types this function accepts. +// Trace adds a TRACE route to the default Mux. See the documentation for +// web.Mux for more information about what types this function accepts. func Trace(pattern interface{}, handler interface{}) { DefaultMux.Trace(pattern, handler) } -// Set the NotFound handler for the default Mux. See the documentation for -// web.Mux.NotFound for more information. +// NotFound sets the NotFound handler for the default Mux. See the documentation +// for web.Mux.NotFound for more information. func NotFound(handler interface{}) { DefaultMux.NotFound(handler) } diff --git a/example/main.go b/example/main.go index 076948b..901f1f4 100644 --- a/example/main.go +++ b/example/main.go @@ -71,8 +71,8 @@ func Root(w http.ResponseWriter, r *http.Request) { } } -// Create a new greet (POST "/greets"). Creates a greet and redirects you to the -// created greet. +// NewGreet creates a new greet (POST "/greets"). Creates a greet and redirects +// you to the created greet. // // To post a new greet, try this at a shell: // $ now=$(date +'%Y-%m-%mT%H:%M:%SZ') @@ -96,7 +96,7 @@ func NewGreet(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, url, http.StatusCreated) } -// Get a given user and her greets (GET "/user/:name") +// GetUser finds a given user and her greets (GET "/user/:name") func GetUser(c web.C, w http.ResponseWriter, r *http.Request) { io.WriteString(w, "Gritter\n======\n\n") handle := c.UrlParams["name"] @@ -116,8 +116,8 @@ func GetUser(c web.C, w http.ResponseWriter, r *http.Request) { } } -// Get a particular greet by ID (GET "/greet/\d+"). Does no bounds checking, so -// will probably panic. +// GetGreet finds a particular greet by ID (GET "/greet/\d+"). Does no bounds +// checking, so will probably panic. func GetGreet(c web.C, w http.ResponseWriter, r *http.Request) { id, err := strconv.Atoi(c.UrlParams["id"]) if err != nil { @@ -131,17 +131,18 @@ func GetGreet(c web.C, w http.ResponseWriter, r *http.Request) { greet.Write(w) } -// Admin root (GET "/admin/root"). Much secret. Very administrate. Wow. +// AdminRoot is root (GET "/admin/root"). Much secret. Very administrate. Wow. func AdminRoot(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "Gritter\n======\n\nSuper secret admin page!\n") } -// How are we doing? (GET "/admin/finances") +// AdminFinances would answer the question 'How are we doing?' +// (GET "/admin/finances") func AdminFinances(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "Gritter\n======\n\nWe're broke! :(\n") } -// 404 handler. +// NotFound is a 404 handler. func NotFound(w http.ResponseWriter, r *http.Request) { http.Error(w, "Umm... have you tried turning it off and on again?", 404) } diff --git a/example/middleware.go b/example/middleware.go index 1b9a0ab..9652ebb 100644 --- a/example/middleware.go +++ b/example/middleware.go @@ -8,7 +8,7 @@ import ( "github.com/zenazn/goji/web" ) -// Middleware to render responses as plain text. +// PlainText sets the content-type of responses to text/plain. func PlainText(h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") @@ -20,7 +20,7 @@ func PlainText(h http.Handler) http.Handler { // Nobody will ever guess this! const Password = "admin:admin" -// HTTP Basic Auth middleware for super-secret admin page. Shhhh! +// SuperSecure is HTTP Basic Auth middleware for super-secret admin page. Shhhh! func SuperSecure(c *web.C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { auth := r.Header.Get("Authorization") diff --git a/example/models.go b/example/models.go index ef9453c..4c34c08 100644 --- a/example/models.go +++ b/example/models.go @@ -6,7 +6,7 @@ import ( "time" ) -// A greet is a 140-character micro-blogpost that has no resemblance whatsoever +// A Greet is a 140-character micro-blogpost that has no resemblance whatsoever // to the noise a bird makes. type Greet struct { User string `param:"user"` @@ -30,7 +30,8 @@ func (g Greet) Write(w io.Writer) { g.Time.Format(time.UnixDate)) } -// A user +// A User is a person. It may even be someone you know. Or a rabbit. Hard to say +// from here. type User struct { Name, Bio string } diff --git a/goji.go b/goji.go index 8f68ac3..5b8a90d 100644 --- a/goji.go +++ b/goji.go @@ -42,7 +42,7 @@ import ( "github.com/zenazn/goji/graceful" ) -// Start Goji using reasonable defaults. +// Serve starts Goji using reasonable defaults. func Serve() { if !flag.Parsed() { flag.Parse() diff --git a/graceful/middleware.go b/graceful/middleware.go index 7f4683f..3e17620 100644 --- a/graceful/middleware.go +++ b/graceful/middleware.go @@ -8,9 +8,10 @@ import ( ) /* -Graceful shutdown middleware. When a graceful shutdown is in progress, this -middleware intercepts responses to add a "Connection: close" header to politely -inform the client that we are about to go away. +Middleware adds graceful shutdown capabilities to the given handler. When a +graceful shutdown is in progress, this middleware intercepts responses to add a +"Connection: close" header to politely inform the client that we are about to go +away. This package creates a shim http.ResponseWriter that it passes to subsequent handlers. Unfortunately, there's a great many optional interfaces that this diff --git a/graceful/net.go b/graceful/net.go index b86af8c..5573796 100644 --- a/graceful/net.go +++ b/graceful/net.go @@ -15,9 +15,9 @@ type gracefulConn interface { gracefulShutdown() } -// Wrap an arbitrary net.Listener for use with graceful shutdowns. All -// net.Conn's Accept()ed by this listener will be auto-wrapped as if WrapConn() -// were called on them. +// WrapListener wraps an arbitrary net.Listener for use with graceful shutdowns. +// All net.Conn's Accept()ed by this listener will be auto-wrapped as if +// WrapConn() were called on them. func WrapListener(l net.Listener) net.Listener { return listener{l} } @@ -32,9 +32,9 @@ func (l listener) Accept() (net.Conn, error) { } /* -Wrap an arbitrary connection for use with graceful shutdowns. The graceful -shutdown process will ensure that this connection is closed before terminating -the process. +WrapConn wraps an arbitrary connection for use with graceful shutdowns. The +graceful shutdown process will ensure that this connection is closed before +terminating the process. In order to use this function, you must call SetReadDeadline() before the call to Read() you might make to read a new request off the wire. The connection is diff --git a/graceful/signal.go b/graceful/signal.go index 9a764a5..1f7d08f 100644 --- a/graceful/signal.go +++ b/graceful/signal.go @@ -31,20 +31,21 @@ func init() { go waitForSignal() } -// Add the given signal to the set of signals that trigger a graceful shutdown. -// Note that for convenience the default interrupt (SIGINT) handler is installed -// at package load time, and unless you call ResetSignals() will be listened for -// in addition to any signals you provide by calling this function. +// AddSignal adds the given signal to the set of signals that trigger a graceful +// shutdown. Note that for convenience the default interrupt (SIGINT) handler is +// installed at package load time, and unless you call ResetSignals() will be +// listened for in addition to any signals you provide by calling this function. func AddSignal(sig ...os.Signal) { signal.Notify(sigchan, sig...) } -// Reset the list of signals that trigger a graceful shutdown. Useful if, for -// instance, you don't want to use the default interrupt (SIGINT) handler. Since -// we necessarily install the SIGINT handler before you have a chance to call -// ResetSignals(), there will be a brief window during which the set of signals -// this package listens for will not be as you intend. Therefore, if you intend -// on using this function, we encourage you to call it as soon as possible. +// ResetSignals resets the list of signals that trigger a graceful shutdown. +// Useful if, for instance, you don't want to use the default interrupt (SIGINT) +// handler. Since we necessarily install the SIGINT handler before you have a +// chance to call ResetSignals(), there will be a brief window during which the +// set of signals this package listens for will not be as you intend. Therefore, +// if you intend on using this function, we encourage you to call it as soon as +// possible. func ResetSignals() { signal.Stop(sigchan) } @@ -56,16 +57,16 @@ func (u userShutdown) String() string { } func (u userShutdown) Signal() {} -// Manually trigger a shutdown from your application. Like Wait(), blocks until -// all connections have gracefully shut down. +// Shutdown manually trigger a shutdown from your application. Like Wait(), +// blocks until all connections have gracefully shut down. func Shutdown() { sigchan <- userShutdown{} <-wait } -// Register a function to be called before any of this package's normal shutdown -// actions. All listeners will be called in the order they were added, from a -// single goroutine. +// PreHook registers a function to be called before any of this package's normal +// shutdown actions. All listeners will be called in the order they were added, +// from a single goroutine. func PreHook(f func()) { hookLock.Lock() defer hookLock.Unlock() @@ -73,9 +74,9 @@ func PreHook(f func()) { prehooks = append(prehooks, f) } -// Register a function to be called after all of this package's normal shutdown -// actions. All listeners will be called in the order they were added, from a -// single goroutine, and are guaranteed to be called after all listening +// PostHook registers a function to be called after all of this package's normal +// shutdown actions. All listeners will be called in the order they were added, +// from a single goroutine, and are guaranteed to be called after all listening // connections have been closed, but before Wait() returns. // // If you've Hijack()ed any connections that must be gracefully shut down in diff --git a/web/middleware/options.go b/web/middleware/options.go index 07437c2..f0f0658 100644 --- a/web/middleware/options.go +++ b/web/middleware/options.go @@ -9,8 +9,8 @@ import ( "github.com/zenazn/goji/web" ) -// Automatically return an appropriate "Allow" header when the request method is -// OPTIONS and the request would have otherwise been 404'd. +// AutomaticOptions automatically return an appropriate "Allow" header when the +// request method is OPTIONS and the request would have otherwise been 404'd. func AutomaticOptions(c *web.C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { // This will probably slow down OPTIONS calls a bunch, but it diff --git a/web/mux.go b/web/mux.go index 33f92c5..f79867f 100644 --- a/web/mux.go +++ b/web/mux.go @@ -5,7 +5,7 @@ import ( ) /* -An HTTP multiplexer, much like net/http's ServeMux. +Mux is an HTTP multiplexer, much like net/http's ServeMux. Routes may be added using any of the various HTTP-method-specific functions. When processing a request, when iterating in insertion order the first route @@ -60,7 +60,7 @@ type Mux struct { var _ http.Handler = &Mux{} var _ Handler = &Mux{} -// Create a new Mux without any routes or middleware. +// New creates a new Mux without any routes or middleware. func New() *Mux { mux := Mux{ mStack: mStack{ @@ -84,8 +84,8 @@ func (m *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) { stack.ServeHTTP(w, r) } -// Serve a context dependent request with the given Mux. Satisfies the -// web.Handler interface. +// ServeHTTPC creates a context dependent request with the given Mux. Satisfies +// the web.Handler interface. func (m *Mux) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { stack := m.mStack.alloc() defer m.mStack.release(stack) diff --git a/web/router.go b/web/router.go index b91dce6..9b7577b 100644 --- a/web/router.go +++ b/web/router.go @@ -61,7 +61,7 @@ type router struct { notFound Handler } -// A pattern determines whether or not a given request matches some criteria. +// A Pattern determines whether or not a given request matches some criteria. // They are often used in routes, which are essentially (pattern, methodSet, // handler) tuples. If the method and pattern match, the given handler is used. // diff --git a/web/web.go b/web/web.go index 8b5e3c1..a766591 100644 --- a/web/web.go +++ b/web/web.go @@ -78,8 +78,8 @@ import ( ) /* -Per-request context object. Threaded through all compliant middleware layers and -to the final request handler. +C is a per-request context object which is threaded through all compliant middleware +layers and to the final request handler. As an implementation detail, references to these structs are reused between requests to reduce allocation churn, but the maps they contain are created fresh @@ -101,22 +101,23 @@ type C struct { Env map[string]interface{} } -// A superset of net/http's http.Handler, which also includes a mechanism for -// serving requests with a context. If your handler does not support the use of -// contexts, we encourage you to use http.Handler instead. +// Handler is a superset of net/http's http.Handler, which also includes a +// mechanism for serving requests with a context. If your handler does not +// support the use of contexts, we encourage you to use http.Handler instead. type Handler interface { http.Handler ServeHTTPC(C, http.ResponseWriter, *http.Request) } -// Like net/http's http.HandlerFunc, but supports a context object. Implements -// both http.Handler and web.Handler free of charge. +// HandlerFunc is like net/http's http.HandlerFunc, but supports a context +// object. Implements both http.Handler and web.Handler free of charge. type HandlerFunc func(C, http.ResponseWriter, *http.Request) func (h HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { h(C{}, w, r) } +// ServeHTTPC wraps ServeHTTP with a context parameter. func (h HandlerFunc) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { h(c, w, r) } From c413a4780fdff6d48daa1204e7b8c2cb1d4576ab Mon Sep 17 00:00:00 2001 From: Coda Hale Date: Sat, 12 Apr 2014 11:38:41 -0700 Subject: [PATCH 045/217] Use a simplified loop. --- param/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/param/parse.go b/param/parse.go index e59432a..ae00998 100644 --- a/param/parse.go +++ b/param/parse.go @@ -159,7 +159,7 @@ func parseSlice(key, keytail string, values []string, target reflect.Value) { slice := reflect.MakeSlice(t, len(values), len(values)) kp := kpath(key, keytail) - for i, _ := range values { + for i := range values { // We actually cheat a little bit and modify the key so we can // generate better debugging messages later key := fmt.Sprintf("%s[%d]", kp, i) From 0b12f5954e9bae5e2037dea7be44dece09c98bd4 Mon Sep 17 00:00:00 2001 From: Coda Hale Date: Sat, 12 Apr 2014 11:39:26 -0700 Subject: [PATCH 046/217] Change requestId to requestID. --- web/middleware/logger.go | 18 +++++++++--------- web/middleware/recoverer.go | 10 +++++----- web/middleware/request_id.go | 23 +++++++++++------------ 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/web/middleware/logger.go b/web/middleware/logger.go index 4fbb0e9..ca4773b 100644 --- a/web/middleware/logger.go +++ b/web/middleware/logger.go @@ -17,9 +17,9 @@ import ( // Logger prints a request ID if one is provided. func Logger(c *web.C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - reqId := GetReqId(*c) + reqID := GetReqID(*c) - printStart(reqId, r) + printStart(reqID, r) lw := wrapWriter(w) @@ -28,17 +28,17 @@ func Logger(c *web.C, h http.Handler) http.Handler { lw.maybeWriteHeader() t2 := time.Now() - printEnd(reqId, lw, t2.Sub(t1)) + printEnd(reqID, lw, t2.Sub(t1)) } return http.HandlerFunc(fn) } -func printStart(reqId string, r *http.Request) { +func printStart(reqID string, r *http.Request) { var buf bytes.Buffer - if reqId != "" { - cW(&buf, bBlack, "[%s] ", reqId) + if reqID != "" { + cW(&buf, bBlack, "[%s] ", reqID) } buf.WriteString("Started ") cW(&buf, bMagenta, "%s ", r.Method) @@ -49,11 +49,11 @@ func printStart(reqId string, r *http.Request) { log.Print(buf.String()) } -func printEnd(reqId string, w writerProxy, dt time.Duration) { +func printEnd(reqID string, w writerProxy, dt time.Duration) { var buf bytes.Buffer - if reqId != "" { - cW(&buf, bBlack, "[%s] ", reqId) + if reqID != "" { + cW(&buf, bBlack, "[%s] ", reqID) } buf.WriteString("Returning ") if w.status() < 200 { diff --git a/web/middleware/recoverer.go b/web/middleware/recoverer.go index adf1ca4..c9b7ee5 100644 --- a/web/middleware/recoverer.go +++ b/web/middleware/recoverer.go @@ -16,11 +16,11 @@ import ( // Recoverer prints a request ID if one is provided. func Recoverer(c *web.C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - reqId := GetReqId(*c) + reqID := GetReqID(*c) defer func() { if err := recover(); err != nil { - printPanic(reqId, err) + printPanic(reqID, err) debug.PrintStack() http.Error(w, http.StatusText(500), 500) } @@ -32,11 +32,11 @@ func Recoverer(c *web.C, h http.Handler) http.Handler { return http.HandlerFunc(fn) } -func printPanic(reqId string, err interface{}) { +func printPanic(reqID string, err interface{}) { var buf bytes.Buffer - if reqId != "" { - cW(&buf, bBlack, "[%s] ", reqId) + if reqID != "" { + cW(&buf, bBlack, "[%s] ", reqID) } cW(&buf, bRed, "panic: %#v", err) diff --git a/web/middleware/request_id.go b/web/middleware/request_id.go index 7c8f5d7..1bbcfe2 100644 --- a/web/middleware/request_id.go +++ b/web/middleware/request_id.go @@ -13,7 +13,7 @@ import ( ) // Key to use when setting the request ID. -const RequestIdKey = "reqId" +const RequestIDKey = "reqID" var prefix string var reqid uint64 @@ -33,18 +33,18 @@ func init() { prefix = fmt.Sprintf("%s/%s", hostname, b64[0:8]) } -// RequestId is a middleware that injects a request ID into the context of each +// RequestID is a middleware that injects a request ID into the context of each // request. A request ID is a string of the form "host.example.com/random-0001", // where "random" is a base62 random string that uniquely identifies this go // process, and where the last number is an atomically incremented request // counter. -func RequestId(c *web.C, h http.Handler) http.Handler { +func RequestID(c *web.C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { if c.Env == nil { c.Env = make(map[string]interface{}) } myid := atomic.AddUint64(&reqid, 1) - c.Env[RequestIdKey] = fmt.Sprintf("%s-%06d", prefix, myid) + c.Env[RequestIDKey] = fmt.Sprintf("%s-%06d", prefix, myid) h.ServeHTTP(w, r) } @@ -52,19 +52,18 @@ func RequestId(c *web.C, h http.Handler) http.Handler { return http.HandlerFunc(fn) } -// Get a request ID from the given context if one is present. Returns the empty -// string if a request ID cannot be found. -func GetReqId(c web.C) string { +// GetReqID returns a request ID from the given context if one is present. +// Returns the empty string if a request ID cannot be found. +func GetReqID(c web.C) string { if c.Env == nil { return "" } - v, ok := c.Env[RequestIdKey] + v, ok := c.Env[RequestIDKey] if !ok { return "" } - if reqId, ok := v.(string); ok { - return reqId - } else { - return "" + if reqID, ok := v.(string); ok { + return reqID } + return "" } From a412c1af776878cbc9ea9686c8ad5b6a7a137f54 Mon Sep 17 00:00:00 2001 From: Coda Hale Date: Sat, 12 Apr 2014 11:40:02 -0700 Subject: [PATCH 047/217] Remove return-only else branches. --- web/middleware/options.go | 3 +-- web/middleware/writer_proxy.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/web/middleware/options.go b/web/middleware/options.go index f0f0658..4bb7b12 100644 --- a/web/middleware/options.go +++ b/web/middleware/options.go @@ -58,9 +58,8 @@ func getValidMethods(c web.C) []string { } if methods, ok := v.([]string); ok { return methods - } else { - return nil } + return nil } // Assumption: the list of methods is teensy, and that anything we could diff --git a/web/middleware/writer_proxy.go b/web/middleware/writer_proxy.go index f9cfe94..bd6c097 100644 --- a/web/middleware/writer_proxy.go +++ b/web/middleware/writer_proxy.go @@ -16,9 +16,8 @@ func wrapWriter(w http.ResponseWriter) writerProxy { bw := basicWriter{ResponseWriter: w} if cn && fl && hj && rf { return &fancyWriter{bw} - } else { - return &bw } + return &bw } type writerProxy interface { From d116ca0e9b3dc4a5f83ccdfd098aed9b266aa4c2 Mon Sep 17 00:00:00 2001 From: Coda Hale Date: Sat, 12 Apr 2014 11:42:42 -0700 Subject: [PATCH 048/217] Move type assertions into tests. --- web/mux.go | 4 ---- web/mux_test.go | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/web/mux.go b/web/mux.go index f79867f..ed087e3 100644 --- a/web/mux.go +++ b/web/mux.go @@ -56,10 +56,6 @@ type Mux struct { router } -// Sanity check types -var _ http.Handler = &Mux{} -var _ Handler = &Mux{} - // New creates a new Mux without any routes or middleware. func New() *Mux { mux := Mux{ diff --git a/web/mux_test.go b/web/mux_test.go index 4eee7cd..6e6619c 100644 --- a/web/mux_test.go +++ b/web/mux_test.go @@ -6,6 +6,10 @@ import ( "testing" ) +// Sanity check types +var _ http.Handler = &Mux{} +var _ Handler = &Mux{} + // There's... really not a lot to do here. func TestIfItWorks(t *testing.T) { From 215a04397ba34e3ba90dd66509bb40b868c94a4c Mon Sep 17 00:00:00 2001 From: Coda Hale Date: Sat, 12 Apr 2014 11:44:47 -0700 Subject: [PATCH 049/217] Change Url to URL. --- web/pattern.go | 12 ++++++------ web/web.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/web/pattern.go b/web/pattern.go index 401363d..9599bfb 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -29,11 +29,11 @@ func (p regexpPattern) Match(r *http.Request, c *C, dryrun bool) bool { return true } - if c.UrlParams == nil { - c.UrlParams = make(map[string]string, len(matches)-1) + if c.URLParams == nil { + c.URLParams = make(map[string]string, len(matches)-1) } for i := 1; i < len(matches); i++ { - c.UrlParams[p.names[i]] = matches[i] + c.URLParams[p.names[i]] = matches[i] } return true } @@ -185,11 +185,11 @@ func (s stringPattern) Match(r *http.Request, c *C, dryrun bool) bool { return true } - if c.UrlParams == nil && len(matches) > 0 { - c.UrlParams = make(map[string]string, len(matches)-1) + if c.URLParams == nil && len(matches) > 0 { + c.URLParams = make(map[string]string, len(matches)-1) } for i, match := range matches { - c.UrlParams[s.pats[i]] = match + c.URLParams[s.pats[i]] = match } return true } diff --git a/web/web.go b/web/web.go index a766591..b590d8b 100644 --- a/web/web.go +++ b/web/web.go @@ -93,7 +93,7 @@ type C struct { // strings that matched those identifiers, but if a unnamed regex // capture is used, it will be assigned to the special identifiers "$1", // "$2", etc. - UrlParams map[string]string + URLParams map[string]string // A free-form environment, similar to Rack or PEP 333's environments. // Middleware layers are encouraged to pass data to downstream layers // and other handlers using this map, and are even more strongly From 19b59fa39dc56222d2baac02543b66c7b687d018 Mon Sep 17 00:00:00 2001 From: Coda Hale Date: Sat, 12 Apr 2014 11:45:08 -0700 Subject: [PATCH 050/217] Change Http to HTTP. --- web/router.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/web/router.go b/web/router.go index 9b7577b..1b77c7a 100644 --- a/web/router.go +++ b/web/router.go @@ -98,14 +98,14 @@ func parsePattern(p interface{}) Pattern { panic("log.Fatalf does not return") } -type netHttpWrap struct { +type netHTTPWrap struct { http.Handler } -func (h netHttpWrap) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (h netHTTPWrap) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.Handler.ServeHTTP(w, r) } -func (h netHttpWrap) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { +func (h netHTTPWrap) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { h.Handler.ServeHTTP(w, r) } @@ -114,13 +114,13 @@ func parseHandler(h interface{}) Handler { case Handler: return h.(Handler) case http.Handler: - return netHttpWrap{h.(http.Handler)} + return netHTTPWrap{h.(http.Handler)} case func(c C, w http.ResponseWriter, r *http.Request): f := h.(func(c C, w http.ResponseWriter, r *http.Request)) return HandlerFunc(f) case func(w http.ResponseWriter, r *http.Request): f := h.(func(w http.ResponseWriter, r *http.Request)) - return netHttpWrap{http.HandlerFunc(f)} + return netHTTPWrap{http.HandlerFunc(f)} default: log.Fatalf("Unknown handler type %v. Expected a web.Handler, "+ "a http.Handler, or a function with signature func(C, "+ From 915dde7c7847af07f6203b95bd6ad0f127fbc61a Mon Sep 17 00:00:00 2001 From: Coda Hale Date: Sat, 12 Apr 2014 11:45:42 -0700 Subject: [PATCH 051/217] Use rt instead of m for the router receiver. --- web/router.go | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/web/router.go b/web/router.go index 1b77c7a..22cc1ca 100644 --- a/web/router.go +++ b/web/router.go @@ -213,22 +213,22 @@ doesn't strip this prefix from the path before forwarding it on (e.g., the handler will see the full path, including the "/admin" part), but this functionality can easily be performed by an extra middleware layer. */ -func (m *router) Handle(pattern interface{}, handler interface{}) { - m.handleUntyped(pattern, mALL, handler) +func (rt *router) Handle(pattern interface{}, handler interface{}) { + rt.handleUntyped(pattern, mALL, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is // CONNECT. See the documentation for type Mux for a description of what types // are accepted for pattern and handler. -func (m *router) Connect(pattern interface{}, handler interface{}) { - m.handleUntyped(pattern, mCONNECT, handler) +func (rt *router) Connect(pattern interface{}, handler interface{}) { + rt.handleUntyped(pattern, mCONNECT, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is // DELETE. See the documentation for type Mux for a description of what types // are accepted for pattern and handler. -func (m *router) Delete(pattern interface{}, handler interface{}) { - m.handleUntyped(pattern, mDELETE, handler) +func (rt *router) Delete(pattern interface{}, handler interface{}) { + rt.handleUntyped(pattern, mDELETE, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is @@ -239,50 +239,50 @@ func (m *router) Delete(pattern interface{}, handler interface{}) { // take care of all the fiddly bits for you. If you wish to provide an alternate // implementation of HEAD, you should add a handler explicitly and place it // above your GET handler. -func (m *router) Get(pattern interface{}, handler interface{}) { - m.handleUntyped(pattern, mGET|mHEAD, handler) +func (rt *router) Get(pattern interface{}, handler interface{}) { + rt.handleUntyped(pattern, mGET|mHEAD, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is // HEAD. See the documentation for type Mux for a description of what types are // accepted for pattern and handler. -func (m *router) Head(pattern interface{}, handler interface{}) { - m.handleUntyped(pattern, mHEAD, handler) +func (rt *router) Head(pattern interface{}, handler interface{}) { + rt.handleUntyped(pattern, mHEAD, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is // OPTIONS. See the documentation for type Mux for a description of what types // are accepted for pattern and handler. -func (m *router) Options(pattern interface{}, handler interface{}) { - m.handleUntyped(pattern, mOPTIONS, handler) +func (rt *router) Options(pattern interface{}, handler interface{}) { + rt.handleUntyped(pattern, mOPTIONS, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is // PATCH. See the documentation for type Mux for a description of what types are // accepted for pattern and handler. -func (m *router) Patch(pattern interface{}, handler interface{}) { - m.handleUntyped(pattern, mPATCH, handler) +func (rt *router) Patch(pattern interface{}, handler interface{}) { + rt.handleUntyped(pattern, mPATCH, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is // POST. See the documentation for type Mux for a description of what types are // accepted for pattern and handler. -func (m *router) Post(pattern interface{}, handler interface{}) { - m.handleUntyped(pattern, mPOST, handler) +func (rt *router) Post(pattern interface{}, handler interface{}) { + rt.handleUntyped(pattern, mPOST, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is // PUT. See the documentation for type Mux for a description of what types are // accepted for pattern and handler. -func (m *router) Put(pattern interface{}, handler interface{}) { - m.handleUntyped(pattern, mPUT, handler) +func (rt *router) Put(pattern interface{}, handler interface{}) { + rt.handleUntyped(pattern, mPUT, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is // TRACE. See the documentation for type Mux for a description of what types are // accepted for pattern and handler. -func (m *router) Trace(pattern interface{}, handler interface{}) { - m.handleUntyped(pattern, mTRACE, handler) +func (rt *router) Trace(pattern interface{}, handler interface{}) { + rt.handleUntyped(pattern, mTRACE, handler) } // Set the fallback (i.e., 404) handler for this mux. See the documentation for @@ -292,6 +292,6 @@ func (m *router) Trace(pattern interface{}, handler interface{}) { // (also available as the constant ValidMethodsKey) will be set to the list of // HTTP methods that could have been routed had they been provided on an // otherwise identical request. -func (m *router) NotFound(handler interface{}) { - m.notFound = parseHandler(handler) +func (rt *router) NotFound(handler interface{}) { + rt.notFound = parseHandler(handler) } From 6be0a4270afe6b0381cd6f82b777a90a3ee9f6eb Mon Sep 17 00:00:00 2001 From: Coda Hale Date: Sat, 12 Apr 2014 11:46:02 -0700 Subject: [PATCH 052/217] No need to re-document methods. --- web/mux.go | 1 - 1 file changed, 1 deletion(-) diff --git a/web/mux.go b/web/mux.go index ed087e3..fcc67e8 100644 --- a/web/mux.go +++ b/web/mux.go @@ -72,7 +72,6 @@ func New() *Mux { return &mux } -// Serve a request with the given Mux. Satisfies the http.Handler interface. func (m *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) { stack := m.mStack.alloc() defer m.mStack.release(stack) From 53f6b7b4a874d42cff3fba5a604cbe9076b67193 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 12 Apr 2014 20:41:07 +0100 Subject: [PATCH 053/217] Minor cosmetic fixes --- graceful/signal.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graceful/signal.go b/graceful/signal.go index 1f7d08f..c231a80 100644 --- a/graceful/signal.go +++ b/graceful/signal.go @@ -39,7 +39,7 @@ func AddSignal(sig ...os.Signal) { signal.Notify(sigchan, sig...) } -// ResetSignals resets the list of signals that trigger a graceful shutdown. +// ResetSignals resets the list of signals that trigger a graceful shutdown. // Useful if, for instance, you don't want to use the default interrupt (SIGINT) // handler. Since we necessarily install the SIGINT handler before you have a // chance to call ResetSignals(), there will be a brief window during which the @@ -57,7 +57,7 @@ func (u userShutdown) String() string { } func (u userShutdown) Signal() {} -// Shutdown manually trigger a shutdown from your application. Like Wait(), +// Shutdown manually triggers a shutdown from your application. Like Wait(), // blocks until all connections have gracefully shut down. func Shutdown() { sigchan <- userShutdown{} From 4cbff375378b10654882363bc38c353efeebeb13 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 12 Apr 2014 20:55:25 +0100 Subject: [PATCH 054/217] Fix up URLParams rename compilation errors This fixes up the last few compilation errors caused by the UrlParams => URLParams rename. --- default.go | 2 +- example/main.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/default.go b/default.go index 5d78ca8..541830e 100644 --- a/default.go +++ b/default.go @@ -11,7 +11,7 @@ var DefaultMux *web.Mux func init() { DefaultMux = web.New() - DefaultMux.Use(middleware.RequestId) + DefaultMux.Use(middleware.RequestID) DefaultMux.Use(middleware.Logger) DefaultMux.Use(middleware.Recoverer) DefaultMux.Use(middleware.AutomaticOptions) diff --git a/example/main.go b/example/main.go index 901f1f4..c4b331a 100644 --- a/example/main.go +++ b/example/main.go @@ -99,7 +99,7 @@ func NewGreet(w http.ResponseWriter, r *http.Request) { // GetUser finds a given user and her greets (GET "/user/:name") func GetUser(c web.C, w http.ResponseWriter, r *http.Request) { io.WriteString(w, "Gritter\n======\n\n") - handle := c.UrlParams["name"] + handle := c.URLParams["name"] user, ok := Users[handle] if !ok { http.Error(w, http.StatusText(404), 404) @@ -119,7 +119,7 @@ func GetUser(c web.C, w http.ResponseWriter, r *http.Request) { // GetGreet finds a particular greet by ID (GET "/greet/\d+"). Does no bounds // checking, so will probably panic. func GetGreet(c web.C, w http.ResponseWriter, r *http.Request) { - id, err := strconv.Atoi(c.UrlParams["id"]) + id, err := strconv.Atoi(c.URLParams["id"]) if err != nil { http.Error(w, http.StatusText(404), 404) return From 3eb0254943392452ceddbb7b310e5c1c1fd4b319 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 12 Apr 2014 21:01:20 +0100 Subject: [PATCH 055/217] Clarify /admin/ vs /admin in example app I'm not particularly satisfied with the solution here. Maybe the answer is "more middleware," but I'm not sure. --- example/main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/example/main.go b/example/main.go index c4b331a..0fa57b2 100644 --- a/example/main.go +++ b/example/main.go @@ -46,9 +46,13 @@ func main() { goji.Handle("/admin/*", admin) admin.Use(SuperSecure) + // Goji's routing, like Sinatra's, is exact: no effort is made to + // normalize trailing slashes. + goji.Get("/admin", http.RedirectHandler("/admin/", 301)) + // Set up admin routes. Note that sub-routes do *not* mutate the path in - // any way, so we need to supply full ("/admin" prefixed) paths. - admin.Get("/admin", AdminRoot) + // any way, so we need to supply full ("/admin/" prefixed) paths. + admin.Get("/admin/", AdminRoot) admin.Get("/admin/finances", AdminFinances) // Use a custom 404 handler From 5d911931d53442bb09b7e0033bc58600042673ec Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 12 Apr 2014 21:04:45 +0100 Subject: [PATCH 056/217] Also fix URLParams in tests --- web/middleware_test.go | 2 +- web/mux_test.go | 2 +- web/pattern_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/middleware_test.go b/web/middleware_test.go index 548e9b9..4e68c41 100644 --- a/web/middleware_test.go +++ b/web/middleware_test.go @@ -214,7 +214,7 @@ func TestContext(t *testing.T) { } st.Use(func(c *C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - if c.Env != nil || c.UrlParams != nil { + if c.Env != nil || c.URLParams != nil { t.Error("Expected a clean context") } c.Env = make(map[string]interface{}) diff --git a/web/mux_test.go b/web/mux_test.go index 6e6619c..1854524 100644 --- a/web/mux_test.go +++ b/web/mux_test.go @@ -25,7 +25,7 @@ func TestIfItWorks(t *testing.T) { greeting = g.(string) } } - ch <- greeting + c.UrlParams["name"] + ch <- greeting + c.URLParams["name"] }) r, _ := http.NewRequest("GET", "/hello/carl", nil) diff --git a/web/pattern_test.go b/web/pattern_test.go index 0377ab0..2ea3c90 100644 --- a/web/pattern_test.go +++ b/web/pattern_test.go @@ -17,7 +17,7 @@ func pt(url string, match bool, params map[string]string) patternTest { r: req, match: match, c: &C{}, - cout: &C{UrlParams: params}, + cout: &C{URLParams: params}, } } From 0e34b28fd108b46bb9870a56d113f87b2e35cc23 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 12 Apr 2014 21:05:32 +0100 Subject: [PATCH 057/217] May as well rename this string too --- web/middleware_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/middleware_test.go b/web/middleware_test.go index 4e68c41..e783427 100644 --- a/web/middleware_test.go +++ b/web/middleware_test.go @@ -203,7 +203,7 @@ func TestInvalidation(t *testing.T) { func TestContext(t *testing.T) { router := func(c C, w http.ResponseWriter, r *http.Request) { - if c.Env["reqId"].(int) != 2 { + if c.Env["reqID"].(int) != 2 { t.Error("Request id was not 2 :(") } } @@ -218,7 +218,7 @@ func TestContext(t *testing.T) { t.Error("Expected a clean context") } c.Env = make(map[string]interface{}) - c.Env["reqId"] = 1 + c.Env["reqID"] = 1 h.ServeHTTP(w, r) } @@ -230,7 +230,7 @@ func TestContext(t *testing.T) { if c.Env == nil { t.Error("Expected env from last middleware") } - c.Env["reqId"] = c.Env["reqId"].(int) + 1 + c.Env["reqID"] = c.Env["reqID"].(int) + 1 h.ServeHTTP(w, r) } From ced741fddf0e76be0737222f421c22288e394af8 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 12 Apr 2014 21:05:55 +0100 Subject: [PATCH 058/217] Update documentation for URLParams rename --- README.md | 2 +- goji.go | 2 +- web/router.go | 2 +- web/web.go | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0120cbc..63022a9 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ import ( ) func hello(c web.C, w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "Hello, %s!", c.UrlParams["name"]) + fmt.Fprintf(w, "Hello, %s!", c.URLParams["name"]) } func main() { diff --git a/goji.go b/goji.go index 5b8a90d..55504d3 100644 --- a/goji.go +++ b/goji.go @@ -13,7 +13,7 @@ Example: ) func hello(c web.C, w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "Hello, %s!", c.UrlParams["name"]) + fmt.Fprintf(w, "Hello, %s!", c.URLParams["name"]) } func main() { diff --git a/web/router.go b/web/router.go index 22cc1ca..e3805ff 100644 --- a/web/router.go +++ b/web/router.go @@ -78,7 +78,7 @@ type Pattern interface { // Returns true if the request satisfies the pattern. This function is // free to examine both the request and the context to make this // decision. After it is certain that the request matches, this function - // should mutate or create c.UrlParams if necessary, unless dryrun is + // should mutate or create c.URLParams if necessary, unless dryrun is // set. Match(r *http.Request, c *C, dryrun bool) bool } diff --git a/web/web.go b/web/web.go index b590d8b..8e49a9d 100644 --- a/web/web.go +++ b/web/web.go @@ -31,11 +31,11 @@ Use your favorite HTTP verbs: Bind parameters using either Sinatra-like patterns or regular expressions: m.Get("/hello/:name", func(c web.C, w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "Hello, %s!", c.UrlParams["name"]) + fmt.Fprintf(w, "Hello, %s!", c.URLParams["name"]) }) pattern := regexp.MustCompile(`^/ip/(?P(?:\d{1,3}\.){3}\d{1,3})$`) m.Get(pattern, func(c web.C, w http.ResponseWriter, r *http.Request) { - fmt.Printf(w, "Info for IP address %s:", c.UrlParams["ip"]) + fmt.Printf(w, "Info for IP address %s:", c.URLParams["ip"]) }) Middleware are functions that wrap http.Handlers, just like you'd use with raw @@ -84,7 +84,7 @@ layers and to the final request handler. As an implementation detail, references to these structs are reused between requests to reduce allocation churn, but the maps they contain are created fresh on every request. If you are closing over a context (especially relevant for -middleware), you should not close over either the UrlParams or Env objects, +middleware), you should not close over either the URLParams or Env objects, instead accessing them through the context whenever they are required. */ type C struct { From 2425c4144d26125636211a605000300a24b89957 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 12 Apr 2014 21:10:30 +0100 Subject: [PATCH 059/217] Fix Error => Errorf This bugfix brought to you by `go vet` --- web/router_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/router_test.go b/web/router_test.go index 5e4d7cd..b8556ce 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -170,7 +170,7 @@ func TestPrefix(t *testing.T) { select { case val := <-ch: if val != "/hello/world" { - t.Error("Got %q, expected /hello/world", val) + t.Errorf("Got %q, expected /hello/world", val) } case <-time.After(5 * time.Millisecond): t.Errorf("Timeout waiting for hello") From eecdf6899d36e57b72c4096ecae98b20e91aa97c Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 13 Apr 2014 19:51:01 +0100 Subject: [PATCH 060/217] Make graceful.Server public again graceful.Server was made private in 05c2ca7e, but I think the increased flexibility you get with being able to provide your own TLS options (etc.) outweighs the API complexity of an additional type. --- graceful/graceful.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/graceful/graceful.go b/graceful/graceful.go index c6dc314..13537e7 100644 --- a/graceful/graceful.go +++ b/graceful/graceful.go @@ -37,9 +37,11 @@ Since I couldn't come up with a better idea, I just copy-and-pasted both ListenAndServe and ListenAndServeTLS here more-or-less verbatim. "Oh well!" */ -type server http.Server +// Type Server is exactly the same as an http.Server, but provides more graceful +// implementations of its methods. +type Server http.Server -func (srv *server) Serve(l net.Listener) (err error) { +func (srv *Server) Serve(l net.Listener) (err error) { go func() { <-kill l.Close() @@ -71,7 +73,7 @@ func (srv *server) Serve(l net.Listener) (err error) { // About 200 years, also known as "forever" const forever time.Duration = 200 * 365 * 24 * time.Hour -func (srv *server) ListenAndServe() error { +func (srv *Server) ListenAndServe() error { addr := srv.Addr if addr == "" { addr = ":http" @@ -83,7 +85,7 @@ func (srv *server) ListenAndServe() error { return srv.Serve(l) } -func (srv *server) ListenAndServeTLS(certFile, keyFile string) error { +func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { addr := srv.Addr if addr == "" { addr = ":https" @@ -114,18 +116,18 @@ func (srv *server) ListenAndServeTLS(certFile, keyFile string) error { // ListenAndServe behaves exactly like the net/http function of the same name. func ListenAndServe(addr string, handler http.Handler) error { - server := &server{Addr: addr, Handler: handler} + server := &Server{Addr: addr, Handler: handler} return server.ListenAndServe() } // ListenAndServeTLS behaves exactly like the net/http function of the same name. func ListenAndServeTLS(addr, certfile, keyfile string, handler http.Handler) error { - server := &server{Addr: addr, Handler: handler} + server := &Server{Addr: addr, Handler: handler} return server.ListenAndServeTLS(certfile, keyfile) } // Serve behaves exactly like the net/http function of the same name. func Serve(l net.Listener, handler http.Handler) error { - server := &server{Handler: handler} + server := &Server{Handler: handler} return server.Serve(l) } From 7b91ca180d5e05fd0b48fbf1645b9fc44e81bd6d Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 21 Apr 2014 18:17:48 +0200 Subject: [PATCH 061/217] gitignore "example" binary --- example/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 example/.gitignore diff --git a/example/.gitignore b/example/.gitignore new file mode 100644 index 0000000..33a9488 --- /dev/null +++ b/example/.gitignore @@ -0,0 +1 @@ +example From bc3ac1d667a277085af337244fdcc2dccd45b011 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 21 Apr 2014 18:18:19 +0200 Subject: [PATCH 062/217] Rewrite AutomaticOptions to not use httptest httptest was adding an extra flag, which was sort of ugly. Instead, reimplement the parts of its functionality we were using. Bonus: due to specialization, it's now a bit more efficient as well! --- web/middleware/options.go | 85 +++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/web/middleware/options.go b/web/middleware/options.go index 4bb7b12..4bdce5f 100644 --- a/web/middleware/options.go +++ b/web/middleware/options.go @@ -1,48 +1,73 @@ package middleware import ( - "io" "net/http" - "net/http/httptest" "strings" "github.com/zenazn/goji/web" ) -// AutomaticOptions automatically return an appropriate "Allow" header when the -// request method is OPTIONS and the request would have otherwise been 404'd. -func AutomaticOptions(c *web.C, h http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - // This will probably slow down OPTIONS calls a bunch, but it - // probably won't happen too much, and it'll just be hitting the - // 404 route anyways. - var fw *httptest.ResponseRecorder - pw := w - if r.Method == "OPTIONS" { - fw = httptest.NewRecorder() - pw = fw - } +type autoOptionsState int - h.ServeHTTP(pw, r) +const ( + aosInit autoOptionsState = iota + aosHeaderWritten + aosProxying +) - if fw == nil { - return - } +// I originally used an httptest.ResponseRecorder here, but package httptest +// adds a flag which I'm not particularly eager to expose. This is essentially a +// ResponseRecorder that has been specialized for the purpose at hand to avoid +// the httptest dependency. +type autoOptionsProxy struct { + w http.ResponseWriter + c *web.C + state autoOptionsState +} + +func (p *autoOptionsProxy) Header() http.Header { + return p.w.Header() +} - for k, v := range fw.Header() { - w.Header()[k] = v +func (p *autoOptionsProxy) Write(buf []byte) (int, error) { + switch p.state { + case aosInit: + p.state = aosHeaderWritten + case aosProxying: + return len(buf), nil + } + return p.w.Write(buf) +} + +func (p *autoOptionsProxy) WriteHeader(code int) { + methods := getValidMethods(*p.c) + switch p.state { + case aosInit: + if methods != nil && code == http.StatusNotFound { + p.state = aosProxying + break } + p.state = aosHeaderWritten + fallthrough + default: + p.w.WriteHeader(code) + return + } - methods := getValidMethods(*c) + methods = addMethod(methods, "OPTIONS") + p.w.Header().Set("Allow", strings.Join(methods, ", ")) + p.w.WriteHeader(http.StatusOK) +} - if fw.Code == http.StatusNotFound && methods != nil { - methods = addMethod(methods, "OPTIONS") - w.Header().Set("Allow", strings.Join(methods, ", ")) - w.WriteHeader(http.StatusOK) - } else { - w.WriteHeader(fw.Code) - io.Copy(w, fw.Body) +// AutomaticOptions automatically return an appropriate "Allow" header when the +// request method is OPTIONS and the request would have otherwise been 404'd. +func AutomaticOptions(c *web.C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if r.Method == "OPTIONS" { + w = &autoOptionsProxy{c: c, w: w} } + + h.ServeHTTP(w, r) } return http.HandlerFunc(fn) @@ -62,8 +87,6 @@ func getValidMethods(c web.C) []string { return nil } -// Assumption: the list of methods is teensy, and that anything we could -// possibly want to do here is going to be fast. func addMethod(methods []string, method string) []string { for _, m := range methods { if m == method { From f8ba05d99a3b55393e63542cefbb07fa0dbbd1fc Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 21 Apr 2014 19:44:00 +0200 Subject: [PATCH 063/217] Options middleware tests They're really verbose. I should probably DRY them up later. --- web/middleware/options_test.go | 112 +++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 web/middleware/options_test.go diff --git a/web/middleware/options_test.go b/web/middleware/options_test.go new file mode 100644 index 0000000..74f5481 --- /dev/null +++ b/web/middleware/options_test.go @@ -0,0 +1,112 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/zenazn/goji/web" +) + +func testOptions(r *http.Request, f func(*web.C, http.ResponseWriter, *http.Request)) *httptest.ResponseRecorder { + var c web.C + + h := func(w http.ResponseWriter, r *http.Request) { + f(&c, w, r) + } + m := AutomaticOptions(&c, http.HandlerFunc(h)) + w := httptest.NewRecorder() + m.ServeHTTP(w, r) + + return w +} + +var optionsTestEnv = map[string]interface{}{ + web.ValidMethodsKey: []string{ + "hello", + "world", + }, +} + +func TestAutomaticOptions(t *testing.T) { + t.Parallel() + + // Shouldn't interfere with normal requests + r, _ := http.NewRequest("GET", "/", nil) + rr := testOptions(r, + func(c *web.C, w http.ResponseWriter, r *http.Request) { + w.Write([]byte{'h', 'i'}) + }, + ) + if rr.Code != http.StatusOK { + t.Errorf("status is %d, not 200", rr.Code) + } + if rr.Body.String() != "hi" { + t.Errorf("body was %q, should be %q", rr.Body.String(), "hi") + } + allow := rr.HeaderMap.Get("Allow") + if allow != "" { + t.Errorf("Allow header was set to %q, should be empty", allow) + } + + // If we respond non-404 to an OPTIONS request, also don't interfere + r, _ = http.NewRequest("OPTIONS", "/", nil) + rr = testOptions(r, + func(c *web.C, w http.ResponseWriter, r *http.Request) { + c.Env = optionsTestEnv + w.Write([]byte{'h', 'i'}) + }, + ) + if rr.Code != http.StatusOK { + t.Errorf("status is %d, not 200", rr.Code) + } + if rr.Body.String() != "hi" { + t.Errorf("body was %q, should be %q", rr.Body.String(), "hi") + } + allow = rr.HeaderMap.Get("Allow") + if allow != "" { + t.Errorf("Allow header was set to %q, should be empty", allow) + } + + // Provide options if we 404. Make sure we nom the output bytes + r, _ = http.NewRequest("OPTIONS", "/", nil) + rr = testOptions(r, + func(c *web.C, w http.ResponseWriter, r *http.Request) { + c.Env = optionsTestEnv + w.WriteHeader(http.StatusNotFound) + w.Write([]byte{'h', 'i'}) + }, + ) + if rr.Code != http.StatusOK { + t.Errorf("status is %d, not 200", rr.Code) + } + if rr.Body.Len() != 0 { + t.Errorf("body was %q, should be empty", rr.Body.String()) + } + allow = rr.HeaderMap.Get("Allow") + correctHeaders := "hello, world, OPTIONS" + if allow != "hello, world, OPTIONS" { + t.Errorf("Allow header should be %q, was %q", correctHeaders, + allow) + } + + // If we somehow 404 without giving a list of valid options, don't do + // anything + r, _ = http.NewRequest("OPTIONS", "/", nil) + rr = testOptions(r, + func(c *web.C, w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte{'h', 'i'}) + }, + ) + if rr.Code != http.StatusNotFound { + t.Errorf("status is %d, not 404", rr.Code) + } + if rr.Body.String() != "hi" { + t.Errorf("body was %q, should be %q", rr.Body.String(), "hi") + } + allow = rr.HeaderMap.Get("Allow") + if allow != "" { + t.Errorf("Allow header was set to %q, should be empty", allow) + } +} From e3ab09de3579969a4d71cc7ccd0ad3962830ff3b Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 21 Apr 2014 20:37:52 +0200 Subject: [PATCH 064/217] Remove key length sanity checking Previously, we disallowed setting the empty string as a key in a map, since at the time it seemed like doing so would allow all sorts of unsavory bugs. In practice, I think this probably isn't actually true, as I wasn't able to think of a scenario in which this bug would materialize during the several moments I thought about it. Plus, the code here to do sanity checking was wrong anyways. --- param/parse.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/param/parse.go b/param/parse.go index ae00998..7239239 100644 --- a/param/parse.go +++ b/param/parse.go @@ -72,9 +72,7 @@ func primitive(tipe, key, keytail string, values []string) { func keyed(tipe reflect.Type, key, keytail string) (string, string) { idx := strings.IndexRune(keytail, ']') - // Keys must be at least 1 rune wide: we refuse to use the empty string - // as the key - if len(keytail) < 3 || keytail[0] != '[' || idx < 2 { + if keytail[0] != '[' || idx == -1 { perr("expected a square bracket delimited index for %q "+ "(of type %v)", kpath(key, keytail), tipe) } From dce7a18ad6bfd62cc16d714ba46843fb00db34ee Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 21 Apr 2014 21:15:09 +0200 Subject: [PATCH 065/217] I spel reel gud --- param/error_helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/param/error_helpers.go b/param/error_helpers.go index 8033c9a..4588e84 100644 --- a/param/error_helpers.go +++ b/param/error_helpers.go @@ -22,7 +22,7 @@ func perr(format string, a ...interface{}) { } // Problem exists between keyboard and chair. This function is used in cases of -// programmer error, i.e. an inappripriate use of the param library, to +// programmer error, i.e. an inappropriate use of the param library, to // immediately force the program to halt with a hopefully helpful error message. func pebkac(format string, a ...interface{}) { err := errors.New(errPrefix + fmt.Sprintf(format, a...) + yourFault) From 356d56a5f73c0c481117c33a9fb45c2acbeb2617 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 22 Apr 2014 11:12:23 +0200 Subject: [PATCH 066/217] Return typed errors in param Instead of returning awkward untyped error strings, return real error structs. This will allow users of the library to extract interesting semantic meaning from our errors, and is just all around less awful than what we had before. --- param/error_helpers.go | 9 ---- param/errors.go | 112 +++++++++++++++++++++++++++++++++++++++++ param/parse.go | 95 +++++++++++++++++++++++----------- param/struct.go | 8 ++- 4 files changed, 184 insertions(+), 40 deletions(-) create mode 100644 param/errors.go diff --git a/param/error_helpers.go b/param/error_helpers.go index 4588e84..9477d3a 100644 --- a/param/error_helpers.go +++ b/param/error_helpers.go @@ -6,21 +6,12 @@ import ( "log" ) -// TODO: someday it might be nice to throw typed errors instead of weird strings - // Testing log.Fatal in tests is... not a thing. Allow tests to stub it out. var pebkacTesting bool const errPrefix = "param/parse: " const yourFault = " This is a bug in your use of the param library." -// Panic with a formatted error. The param library uses panics to quickly unwind -// the call stack and return a user error -func perr(format string, a ...interface{}) { - err := errors.New(errPrefix + fmt.Sprintf(format, a...)) - panic(err) -} - // Problem exists between keyboard and chair. This function is used in cases of // programmer error, i.e. an inappropriate use of the param library, to // immediately force the program to halt with a hopefully helpful error message. diff --git a/param/errors.go b/param/errors.go new file mode 100644 index 0000000..64011e7 --- /dev/null +++ b/param/errors.go @@ -0,0 +1,112 @@ +package param + +import ( + "fmt" + "reflect" +) + +// TypeError is an error type returned when param has difficulty deserializing a +// parameter value. +type TypeError struct { + // The key that was in error. + Key string + // The type that was expected. + Type reflect.Type + // The underlying error produced as part of the deserialization process, + // if one exists. + Err error +} + +func (t TypeError) Error() string { + return fmt.Sprintf("param: error parsing key %q as %v: %v", t.Key, t.Type, + t.Err) +} + +// SingletonError is an error type returned when a parameter is passed multiple +// times when only a single value is expected. For example, for a struct with +// integer field "foo", "foo=1&foo=2" will return a SingletonError with key +// "foo". +type SingletonError struct { + // The key that was in error. + Key string + // The type that was expected for that key. + Type reflect.Type + // The list of values that were provided for that key. + Values []string +} + +func (s SingletonError) Error() string { + return fmt.Sprintf("param: error parsing key %q: expected single "+ + "value but was given %d: %v", s.Key, len(s.Values), s.Values) +} + +// NestingError is an error type returned when a key is nested when the target +// type does not support nesting of the given type. For example, deserializing +// the parameter key "anint[foo]" into a struct that defines an integer param +// "anint" will produce a NestingError with key "anint" and nesting "[foo]". +type NestingError struct { + // The portion of the key that was correctly parsed into a value. + Key string + // The type of the key that was invalidly nested on. + Type reflect.Type + // The portion of the key that could not be parsed due to invalid + // nesting. + Nesting string +} + +func (n NestingError) Error() string { + return fmt.Sprintf("param: error parsing key %q: invalid nesting "+ + "%q on %s key %q", n.Key+n.Nesting, n.Nesting, n.Type, n.Key) +} + +// SyntaxErrorSubtype describes what sort of syntax error was encountered. +type SyntaxErrorSubtype int + +const ( + MissingOpeningBrace SyntaxErrorSubtype = iota + 1 + MissingClosingBrace +) + +// SyntaxError is an error type returned when a key is incorrectly formatted. +type SyntaxError struct { + // The key for which there was a syntax error. + Key string + // The subtype of the syntax error, which describes what sort of error + // was encountered. + Subtype SyntaxErrorSubtype + // The part of the key (generally the suffix) that was in error. + ErrorPart string +} + +func (s SyntaxError) Error() string { + prefix := fmt.Sprintf("param: syntax error while parsing key %q: ", + s.Key) + + switch s.Subtype { + case MissingOpeningBrace: + return prefix + fmt.Sprintf("expected opening brace, got %q", + s.ErrorPart) + case MissingClosingBrace: + return prefix + fmt.Sprintf("expected closing brace in %q", + s.ErrorPart) + default: + panic("switch is not exhaustive!") + } +} + +// KeyError is an error type returned when an unknown field is set on a struct. +type KeyError struct { + // The full key that was in error. + FullKey string + // The key of the struct that did not have the given field. + Key string + // The type of the struct that did not have the given field. + Type reflect.Type + // The name of the field which was not present. + Field string +} + +func (k KeyError) Error() string { + return fmt.Sprintf("param: error parsing key %q: unknown field %q on "+ + "struct %q of type %v", k.FullKey, k.Field, k.Key, k.Type) +} diff --git a/param/parse.go b/param/parse.go index 7239239..1213d8c 100644 --- a/param/parse.go +++ b/param/parse.go @@ -59,39 +59,60 @@ func kpath(key, keytail string) string { // Helper for validating that a value has been passed exactly once, and that the // user is not attempting to nest on the key. -func primitive(tipe, key, keytail string, values []string) { +func primitive(key, keytail string, tipe reflect.Type, values []string) { if keytail != "" { - perr("expected %s for key %q, got nested value", tipe, - kpath(key, keytail)) + panic(NestingError{ + Key: kpath(key, keytail), + Type: tipe, + Nesting: keytail, + }) } if len(values) != 1 { - perr("expected %s for key %q, but key passed %v times", tipe, - kpath(key, keytail), len(values)) + panic(SingletonError{ + Key: kpath(key, keytail), + Type: tipe, + Values: values, + }) } } func keyed(tipe reflect.Type, key, keytail string) (string, string) { + if keytail[0] != '[' { + panic(SyntaxError{ + Key: kpath(key, keytail), + Subtype: MissingOpeningBrace, + ErrorPart: keytail, + }) + } + idx := strings.IndexRune(keytail, ']') - if keytail[0] != '[' || idx == -1 { - perr("expected a square bracket delimited index for %q "+ - "(of type %v)", kpath(key, keytail), tipe) + if idx == -1 { + panic(SyntaxError{ + Key: kpath(key, keytail), + Subtype: MissingClosingBrace, + ErrorPart: keytail[1:], + }) } + return keytail[1:idx], keytail[idx+1:] } func parseTextUnmarshaler(key, keytail string, values []string, target reflect.Value) { - primitive("encoding.TextUnmarshaler", key, keytail, values) + primitive(key, keytail, target.Type(), values) tu := target.Addr().Interface().(encoding.TextUnmarshaler) err := tu.UnmarshalText([]byte(values[0])) if err != nil { - perr("error while calling UnmarshalText on %v for key %q: %v", - target.Type(), kpath(key, keytail), err) + panic(TypeError{ + Key: kpath(key, keytail), + Type: target.Type(), + Err: err, + }) } } func parseBool(key, keytail string, values []string, target reflect.Value) { - primitive("bool", key, keytail, values) + primitive(key, keytail, target.Type(), values) switch values[0] { case "true", "1", "on": @@ -99,61 +120,77 @@ func parseBool(key, keytail string, values []string, target reflect.Value) { case "false", "0", "": target.SetBool(false) default: - perr("could not parse key %q as bool", kpath(key, keytail)) + panic(TypeError{ + Key: kpath(key, keytail), + Type: target.Type(), + }) } } func parseInt(key, keytail string, values []string, target reflect.Value) { - primitive("int", key, keytail, values) - t := target.Type() + primitive(key, keytail, t, values) + i, err := strconv.ParseInt(values[0], 10, t.Bits()) if err != nil { - perr("error parsing key %q as int: %v", kpath(key, keytail), - err) + panic(TypeError{ + Key: kpath(key, keytail), + Type: t, + Err: err, + }) } target.SetInt(i) } func parseUint(key, keytail string, values []string, target reflect.Value) { - primitive("uint", key, keytail, values) - t := target.Type() + primitive(key, keytail, t, values) + i, err := strconv.ParseUint(values[0], 10, t.Bits()) if err != nil { - perr("error parsing key %q as uint: %v", kpath(key, keytail), - err) + panic(TypeError{ + Key: kpath(key, keytail), + Type: t, + Err: err, + }) } target.SetUint(i) } func parseFloat(key, keytail string, values []string, target reflect.Value) { - primitive("float", key, keytail, values) - t := target.Type() + primitive(key, keytail, t, values) + f, err := strconv.ParseFloat(values[0], t.Bits()) if err != nil { - perr("error parsing key %q as float: %v", kpath(key, keytail), - err) + panic(TypeError{ + Key: kpath(key, keytail), + Type: t, + Err: err, + }) } target.SetFloat(f) } func parseString(key, keytail string, values []string, target reflect.Value) { - primitive("string", key, keytail, values) + primitive(key, keytail, target.Type(), values) target.SetString(values[0]) } func parseSlice(key, keytail string, values []string, target reflect.Value) { + t := target.Type() + // BUG(carl): We currently do not handle slices of nested types. If // support is needed, the implementation probably could be fleshed out. if keytail != "[]" { - perr("unexpected array nesting for key %q: %q", - kpath(key, keytail), keytail) + panic(NestingError{ + Key: kpath(key, keytail), + Type: t, + Nesting: keytail, + }) } - t := target.Type() slice := reflect.MakeSlice(t, len(values), len(values)) kp := kpath(key, keytail) diff --git a/param/struct.go b/param/struct.go index 314d5b0..8af3c08 100644 --- a/param/struct.go +++ b/param/struct.go @@ -108,8 +108,12 @@ func extractHandler(s reflect.Type, sf reflect.StructField) func(string, string, func parseStructField(cache structCache, key, sk, keytail string, values []string, target reflect.Value) { l, ok := cache[sk] if !ok { - perr("unknown key %q for struct at key %q", sk, - kpath(key, keytail)) + panic(KeyError{ + FullKey: key, + Key: kpath(key, keytail), + Type: target.Type(), + Field: sk, + }) } f := target.Field(l.offset) From 74e5c940d0fb257177db39c7a6a411ad84bcfd85 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 22 Apr 2014 11:29:52 +0200 Subject: [PATCH 067/217] s/brace/bracket ...oops. "[]" are brackets. "{}" are braces. --- param/errors.go | 12 ++++++------ param/parse.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/param/errors.go b/param/errors.go index 64011e7..e6b9de6 100644 --- a/param/errors.go +++ b/param/errors.go @@ -63,8 +63,8 @@ func (n NestingError) Error() string { type SyntaxErrorSubtype int const ( - MissingOpeningBrace SyntaxErrorSubtype = iota + 1 - MissingClosingBrace + MissingOpeningBracket SyntaxErrorSubtype = iota + 1 + MissingClosingBracket ) // SyntaxError is an error type returned when a key is incorrectly formatted. @@ -83,11 +83,11 @@ func (s SyntaxError) Error() string { s.Key) switch s.Subtype { - case MissingOpeningBrace: - return prefix + fmt.Sprintf("expected opening brace, got %q", + case MissingOpeningBracket: + return prefix + fmt.Sprintf("expected opening bracket, got %q", s.ErrorPart) - case MissingClosingBrace: - return prefix + fmt.Sprintf("expected closing brace in %q", + case MissingClosingBracket: + return prefix + fmt.Sprintf("expected closing bracket in %q", s.ErrorPart) default: panic("switch is not exhaustive!") diff --git a/param/parse.go b/param/parse.go index 1213d8c..b8c069d 100644 --- a/param/parse.go +++ b/param/parse.go @@ -80,7 +80,7 @@ func keyed(tipe reflect.Type, key, keytail string) (string, string) { if keytail[0] != '[' { panic(SyntaxError{ Key: kpath(key, keytail), - Subtype: MissingOpeningBrace, + Subtype: MissingOpeningBracket, ErrorPart: keytail, }) } @@ -89,7 +89,7 @@ func keyed(tipe reflect.Type, key, keytail string) (string, string) { if idx == -1 { panic(SyntaxError{ Key: kpath(key, keytail), - Subtype: MissingClosingBrace, + Subtype: MissingClosingBracket, ErrorPart: keytail[1:], }) } From f525820cc21afcda714ae46327085be240b0b3a5 Mon Sep 17 00:00:00 2001 From: Nick Presta Date: Tue, 22 Apr 2014 15:13:56 -0400 Subject: [PATCH 068/217] Adding TravisCI configuration, badge, GoDoc badge. --- .travis.yml | 11 +++++++++++ README.md | 7 ++----- 2 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..7d3def7 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +language: go +go: + - 1.2 + - tip +install: + - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v + - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v + - go get code.google.com/p/go.tools/cmd/cover + - go build -v ./... +script: + - go test -v -cover ./... diff --git a/README.md b/README.md index 63022a9..e6105a4 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,7 @@ -Goji +Goji [![GoDoc](https://godoc.org/github.com/zenazn/goji?status.png)](https://godoc.org/github.com/zenazn/goji) [![Build Status](https://travis-ci.org/zenazn/goji.svg)](https://travis-ci.org/zenazn/goji) ==== -Goji is a minimalistic web framework inspired by Sinatra. [Godoc][doc]. - -[doc]: http://godoc.org/github.com/zenazn/goji - +Goji is a minimalistic web framework inspired by Sinatra. Example ------- From 0206512031ad8bbe2feffa367d1900637c1cccfc Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 24 Apr 2014 00:44:43 +0200 Subject: [PATCH 069/217] Add Go version compatibility note Fixes #7 --- goji.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/goji.go b/goji.go index 55504d3..80d3ce9 100644 --- a/goji.go +++ b/goji.go @@ -30,6 +30,8 @@ A side effect of this package's ease-of-use is the fact that it is opinionated. If you don't like (or have outgrown) its opinions, it should be straightforward to use the APIs of goji's subpackages to reimplement things to your liking. Both methods of using this library are equally well supported. + +Goji requires Go 1.2 or newer. */ package goji From 60a7a00257396df0cc90b73b075e5e2b5b6244ba Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 27 Apr 2014 00:37:33 +0200 Subject: [PATCH 070/217] Better comments around middleware stack caching --- web/middleware.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/web/middleware.go b/web/middleware.go index 12c6eae..f9e7963 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -22,10 +22,23 @@ type mStack struct { router Handler } -// Constructing a middleware stack involves a lot of allocations: at the very -// least each layer will have to close over the layer after (inside) it, and -// perhaps a context object. Instead of doing this on every request, let's cache -// fully assembled middleware stacks (the "c" stands for "cached"). +/* +Constructing a middleware stack involves a lot of allocations: at the very least +each layer will have to close over the layer after (inside) it, and perhaps a +context object. Instead of doing this on every request, let's cache fully +assembled middleware stacks (the "c" stands for "cached"). + +A lot of the complexity here (in particular the "pool" parameter, and the +behavior of release() and invalidate() below) is due to the fact that when the +middleware stack is mutated we need to create a "cache barrier," where no +cStack created before the middleware stack mutation is returned to the active +cache pool (and is therefore eligible for subsequent reuse). The way we do this +is a bit ugly: each cStack maintains a pointer to the pool it originally came +from, and will only return itself to that pool. If the mStack's pool has been +rotated since then (meaning that this cStack is invalid), it will either try +(and likely fail) to insert itself into the stale pool, or it will drop the +cStack on the floor. +*/ type cStack struct { C m http.Handler From eb08516a5e001c25761b0df161a3205f545a45cd Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 27 Apr 2014 00:54:29 +0200 Subject: [PATCH 071/217] EnvInit middleware Provide a standard middleware to set c.Env. Don't include it in the default stack, however, since the RequestID middleware will end up allocating Env anyways. Fixes #11 --- web/middleware/envinit.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 web/middleware/envinit.go diff --git a/web/middleware/envinit.go b/web/middleware/envinit.go new file mode 100644 index 0000000..aefefb7 --- /dev/null +++ b/web/middleware/envinit.go @@ -0,0 +1,26 @@ +package middleware + +import ( + "net/http" + + "github.com/zenazn/goji/web" +) + +// EnvInit is a middleware that allocates an environment map if one does not +// already exist. This is necessary because Goji does not guarantee that Env is +// present when running middleware (it avoids forcing the map allocation). Note +// that other middleware should check Env for nil in order to maximize +// compatibility (when EnvInit is not used, or when another middleware layer +// blanks out Env), but for situations in which the user controls the middleware +// stack and knows EnvInit is present, this middleware can eliminate a lot of +// boilerplate. +func EnvInit(c *web.C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if c.Env == nil { + c.Env = make(map[string]interface{}) + } + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} From b01cc3dadf54fcf24b1bbc887194a087ce31a12d Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 27 Apr 2014 01:35:52 +0200 Subject: [PATCH 072/217] Add RealIP middleware This middleware allows you to override a http.Request's RemoteAddr with a value derived from either the X-Forwarded-For or X-Real-IP headers. Fixes #12. --- web/middleware/realip.go | 61 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 web/middleware/realip.go diff --git a/web/middleware/realip.go b/web/middleware/realip.go new file mode 100644 index 0000000..99cd311 --- /dev/null +++ b/web/middleware/realip.go @@ -0,0 +1,61 @@ +package middleware + +import ( + "net/http" + "strings" + + "github.com/zenazn/goji/web" +) + +// Key the original value of RemoteAddr is stored under. +const OriginalRemoteAddrKey = "originalRemoteAddr" + +var xForwardedFor = http.CanonicalHeaderKey("X-Forwarded-For") +var xRealIP = http.CanonicalHeaderKey("X-Real-IP") + +// RealIP is a middleware that sets a http.Request's RemoteAddr to the results +// of parsing either the X-Forwarded-For header or the X-Real-IP header (in that +// order). It places the original value of RemoteAddr in a context environment +// variable. +// +// This middleware should be inserted fairly early in the middleware stack to +// ensure that subsequent layers (e.g., request loggers) which examine the +// RemoteAddr will see the intended value. +// +// You should only use this middleware if you can trust the headers passed to +// you (in particular, the two headers this middleware uses), for example +// because you have placed a reverse proxy like HAProxy or nginx in front of +// Goji. If your reverse proxies are configured to pass along arbitrary header +// values from the client, or if you use this middleware without a reverse +// proxy, malicious clients will be able to make you very sad (or, depending on +// how you're using RemoteAddr, vulnerable to an attack of some sort). +func RealIP(c *web.C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if rip := realIP(r); rip != "" { + if c.Env == nil { + c.Env = make(map[string]interface{}) + } + c.Env[OriginalRemoteAddrKey] = r.RemoteAddr + r.RemoteAddr = rip + } + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} + +func realIP(r *http.Request) string { + var ip string + + if xff := r.Header.Get(xForwardedFor); xff != "" { + i := strings.Index(xff, ", ") + if i == -1 { + i = len(xff) + } + ip = xff[:i] + } else if xrip := r.Header.Get(xRealIP); xrip != "" { + ip = xrip + } + + return ip +} From 346d35381a33a4a5badfd9a8e9f461cc89c51673 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 28 Apr 2014 19:34:14 +0100 Subject: [PATCH 073/217] Add route selection test I'm about to make changes to the way routes are selected, so we may as well get good test coverage of that. --- web/router_test.go | 87 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/web/router_test.go b/web/router_test.go index b8556ce..163ea09 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -132,6 +132,93 @@ func TestHandlerTypes(t *testing.T) { } } +// The idea behind this test is to comprehensively test if routes are being +// applied in the right order. We define a special pattern type that always +// matches so long as it's greater than or equal to the global test index. By +// incrementing this index, we can invalidate all routes up to some point, and +// therefore test the routing guarantee that Goji provides: for any path P, if +// both A and B match P, and if A was inserted before B, then Goji will route to +// A before it routes to B. +var rsRoutes = []string{ + "/", + "/a", + "/a", + "/b", + "/ab", + "/", + "/ba", + "/b", + "/a", +} + +var rsTests = []struct { + key string + results []int +}{ + {"/", []int{0, 5, 5, 5, 5, 5, -1, -1, -1, -1}}, + {"/a", []int{0, 1, 2, 5, 5, 5, 8, 8, 8, -1}}, + {"/b", []int{0, 3, 3, 3, 5, 5, 7, 7, -1, -1}}, + {"/ab", []int{0, 1, 2, 4, 4, 5, 8, 8, 8, -1}}, + {"/ba", []int{0, 3, 3, 3, 5, 5, 6, 7, -1, -1}}, + {"/c", []int{0, 5, 5, 5, 5, 5, -1, -1, -1, -1}}, + {"nope", []int{-1, -1, -1, -1, -1, -1, -1, -1, -1, -1}}, +} + +type rsPattern struct { + i int + counter *int + prefix string + ichan chan int +} + +func (rs rsPattern) Prefix() string { + return rs.prefix +} +func (rs rsPattern) Match(_ *http.Request, _ *C, _ bool) bool { + return rs.i >= *rs.counter +} + +func (rs rsPattern) ServeHTTP(_ http.ResponseWriter, _ *http.Request) { + rs.ichan <- rs.i +} + +var _ Pattern = rsPattern{} +var _ http.Handler = rsPattern{} + +func TestRouteSelection(t *testing.T) { + t.Parallel() + rt := makeRouter() + counter := 0 + ichan := make(chan int, 1) + rt.NotFound(func(w http.ResponseWriter, r *http.Request) { + ichan <- -1 + }) + + for i, s := range rsRoutes { + pat := rsPattern{ + i: i, + counter: &counter, + prefix: s, + ichan: ichan, + } + rt.Get(pat, pat) + } + + for _, test := range rsTests { + var n int + for counter, n = range test.results { + r, _ := http.NewRequest("GET", test.key, nil) + w := httptest.NewRecorder() + rt.route(C{}, w, r) + actual := <-ichan + if n != actual { + t.Errorf("Expected %q @ %d to be %d, got %d", + test.key, counter, n, actual) + } + } + } +} + func TestNotFound(t *testing.T) { t.Parallel() rt := makeRouter() From e88b7661bde062e9c654c4c5037fb6449442835d Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 28 Apr 2014 20:04:42 +0100 Subject: [PATCH 074/217] Sort routes Partially sort the routes on insertion. We're doing this so we can do more efficient things to routes later. The sorting rules are a bit subtle since we aren't allowed to rearrange routes in a way that would cause the semantics to differ from the dumb linear scan. --- web/router.go | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/web/router.go b/web/router.go index e3805ff..3bf384d 100644 --- a/web/router.go +++ b/web/router.go @@ -184,17 +184,35 @@ func (rt *router) handleUntyped(p interface{}, m method, h interface{}) { } func (rt *router) handle(p Pattern, m method, h Handler) { - // We're being a little sloppy here: we assume that pointer assignments - // are atomic, and that there is no way a locked append here can affect - // another goroutine which looked at rt.routes without a lock. rt.lock.Lock() defer rt.lock.Unlock() - rt.routes = append(rt.routes, route{ - prefix: p.Prefix(), + + // Calculate the sorted insertion point, because there's no reason to do + // swapping hijinks if we're already making a copy. We need to use + // bubble sort because we can only compare adjacent elements. + pp := p.Prefix() + var i int + for i = len(rt.routes); i > 0; i-- { + rip := rt.routes[i-1].prefix + if rip <= pp || strings.HasPrefix(rip, pp) { + break + } + } + + newRoutes := make([]route, len(rt.routes)+1) + copy(newRoutes, rt.routes[:i]) + newRoutes[i] = route{ + prefix: pp, method: m, pattern: p, handler: h, - }) + } + copy(newRoutes[i+1:], rt.routes[i:]) + + // We're being a bit sloppy here: we assume that pointer assignment is + // atomic with respect to other agents that don't acquire the lock. We + // should really just give up and use sync/atomic for this. + rt.routes = newRoutes } // This is a bit silly, but I've renamed the method receivers in the public From 72c60e267e7fecd9a1a3b4da941b78c398e3faaf Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 3 May 2014 12:08:15 -0700 Subject: [PATCH 075/217] Fast bytecode router Swap out the naive "try all the routes in order" router with a "compile a trie down to bytecode" router. It's a ton faster, while providing all the same semantics. See the documentation at the top of web/fast_router.go for more. --- web/atomic.go | 16 +++ web/fast_router.go | 265 +++++++++++++++++++++++++++++++++++++++++++++ web/router.go | 97 +++++++++++++++-- 3 files changed, 366 insertions(+), 12 deletions(-) create mode 100644 web/atomic.go create mode 100644 web/fast_router.go diff --git a/web/atomic.go b/web/atomic.go new file mode 100644 index 0000000..1bbf48e --- /dev/null +++ b/web/atomic.go @@ -0,0 +1,16 @@ +package web + +import ( + "sync/atomic" + "unsafe" +) + +func (rt *router) getMachine() routeMachine { + ptr := (*unsafe.Pointer)(unsafe.Pointer(&rt.machine)) + sm := (*routeMachine)(atomic.LoadPointer(ptr)) + return *sm +} +func (rt *router) setMachine(m *routeMachine) { + ptr := (*unsafe.Pointer)(unsafe.Pointer(&rt.machine)) + atomic.StorePointer(ptr, unsafe.Pointer(m)) +} diff --git a/web/fast_router.go b/web/fast_router.go new file mode 100644 index 0000000..dea67db --- /dev/null +++ b/web/fast_router.go @@ -0,0 +1,265 @@ +package web + +/* +This file implements a fast router by encoding a list of routes first into a +pseudo-trie, then encoding that pseudo-trie into a state machine realized as +a routing bytecode. + +The most interesting part of this router is not its speed (it is quite fast), +but the guarantees it provides. In a naive router, routes are examined one after +another until a match is found, and this is the programming model we want to +support. For any given request ("GET /hello/carl"), there is a list of +"plausible" routes: routes which match the method ("GET"), and which have a +prefix that is a prefix of the requested path ("/" and "/hello/", for instance, +but not "/foobar"). Patterns also have some amount of arbitrary code associated +with them, which tells us whether or not the route matched. Just like the naive +router, our goal is to call each plausible pattern, in the order they were +added, until we find one that matches. The "fast" part here is being smart about +which non-plausible routes we can skip. + +First, we sort routes using a pairwise comparison function: sorting occurs as +normal on the prefixes, with the caveat that a route may not be moved past a +route that might also match the same string. Among other things, this means +we're forced to use particularly dumb sorting algorithms, but it only has to +happen once, and there probably aren't even that many routes to begin with. This +logic appears inline in the router's handle() function. + +We then build a pseudo-trie from the sorted list of routes. It's not quite a +normal trie because there are certain routes we cannot reorder around other +routes (since we're providing identical semantics to the naive router), but it's +close enough and the basic idea is the same. + +Finally, we lower this psuedo-trie from its tree representation to a state +machine bytecode. The bytecode is pretty simple: it contains up to three bytes, +a choice of a bunch of flags, and an index. The state machine is pretty simple: +if the bytes match the next few bytes after the cursor, the instruction matches, +and the state machine advances to the next instruction. If it does not match, it +jumps to the instruction at the index. Various flags modify this basic behavior, +the documentation for which can be found below. + +The thing we're optimizing for here over pretty much everything else is memory +locality. We make an effort to lay out both the trie child selection logic and +the matching of long strings consecutively in memory, making both operations +very cheap. In fact, our matching logic isn't particularly asymptotically good, +but in practice the benefits of memory locality outweigh just about everything +else. + +Unfortunately, the code implementing all of this is pretty bad (both inefficient +and hard to read). Maybe someday I'll come and take a second pass at it. +*/ +type state struct { + bs [3]byte + mode smMode + i int32 +} +type stateMachine []state + +type smMode uint8 + +// Many combinations of smModes don't make sense, but since this is interal to +// the library I don't feel like documenting them. +const ( + // The two low bits of the mode are used as a length of how many bytes + // of bs are used. If the length is 0, the node is treated as a + // wildcard. + smLengthMask smMode = 3 +) + +const ( + // Jump to the given index on a match. Ordinarily, the state machine + // will jump to the state given by the index if the characters do not + // match. + smJumpOnMatch smMode = 4 << iota + // The index is the index of a route to try. If running the route fails, + // the state machine advances by one. + smRoute + // Reset the state machine's cursor into the input string to the state's + // index value. + smSetCursor + // If this bit is set, the machine transitions into a non-accepting + // state if it matches. + smFail +) + +type trie struct { + prefix string + children []trieSegment +} + +// A trie segment is a route matching this point (or -1), combined with a list +// of trie children that follow that route. +type trieSegment struct { + route int + children []trie +} + +func buildTrie(routes []route, dp, dr int) trie { + var t trie + ts := trieSegment{-1, nil} + for i, r := range routes { + if len(r.prefix) != dp { + continue + } + + if i == 0 { + ts.route = 0 + } else { + subroutes := routes[ts.route+1 : i] + ts.children = buildTrieSegment(subroutes, dp, dr+ts.route+1) + t.children = append(t.children, ts) + ts = trieSegment{i, nil} + } + } + + // This could be a little DRYer... + subroutes := routes[ts.route+1:] + ts.children = buildTrieSegment(subroutes, dp, dr+ts.route+1) + t.children = append(t.children, ts) + + for i := range t.children { + if t.children[i].route != -1 { + t.children[i].route += dr + } + } + + return t +} + +func commonPrefix(s1, s2 string) string { + if len(s1) > len(s2) { + return commonPrefix(s2, s1) + } + for i := 0; i < len(s1); i++ { + if s1[i] != s2[i] { + return s1[:i] + } + } + return s1 +} + +func buildTrieSegment(routes []route, dp, dr int) []trie { + if len(routes) == 0 { + return nil + } + var tries []trie + + start := 0 + p := routes[0].prefix[dp:] + for i := 1; i < len(routes); i++ { + ip := routes[i].prefix[dp:] + cp := commonPrefix(p, ip) + if len(cp) == 0 { + t := buildTrie(routes[start:i], dp+len(p), dr+start) + t.prefix = p + tries = append(tries, t) + start = i + p = ip + } else { + p = cp + } + } + + t := buildTrie(routes[start:], dp+len(p), dr+start) + t.prefix = p + return append(tries, t) +} + +// This is a bit confusing, since the encode method on a trie deals exclusively +// with trieSegments (i.e., its children), and vice versa. +// +// These methods are also hideously inefficient, both in terms of memory usage +// and algorithmic complexity. If it ever becomes a problem, maybe we can do +// something smarter than stupid O(N^2) appends, but to be honest, I bet N is +// small (it almost always is :P) and we only do it once at boot anyways. + +func (t trie) encode(dp, off int) stateMachine { + ms := make([]stateMachine, len(t.children)) + subs := make([]stateMachine, len(t.children)) + var l, msl, subl int + + for i, ts := range t.children { + ms[i], subs[i] = ts.encode(dp, 0) + msl += len(ms[i]) + l += len(ms[i]) + len(subs[i]) + } + + l++ + + m := make(stateMachine, 0, l) + for i, mm := range ms { + for j := range mm { + if mm[j].mode&(smRoute|smSetCursor) != 0 { + continue + } + + mm[j].i += int32(off + msl + subl + 1) + } + m = append(m, mm...) + subl += len(subs[i]) + } + + m = append(m, state{mode: smJumpOnMatch, i: -1}) + + msl = 0 + for i, sub := range subs { + msl += len(ms[i]) + for j := range sub { + if sub[j].mode&(smRoute|smSetCursor) != 0 { + continue + } + if sub[j].i == -1 { + sub[j].i = int32(off + msl) + } else { + sub[j].i += int32(off + len(m)) + } + } + m = append(m, sub...) + } + + return m +} + +func (ts trieSegment) encode(dp, off int) (me stateMachine, sub stateMachine) { + o := 1 + if ts.route != -1 { + o++ + } + me = make(stateMachine, len(ts.children)+o) + + me[0] = state{mode: smSetCursor, i: int32(dp)} + if ts.route != -1 { + me[1] = state{mode: smRoute, i: int32(ts.route)} + } + + for i, t := range ts.children { + p := t.prefix + + bc := copy(me[i+o].bs[:], p) + me[i+o].mode = smMode(bc) | smJumpOnMatch + me[i+o].i = int32(off + len(sub)) + + for len(p) > bc { + var bs [3]byte + p = p[bc:] + bc = copy(bs[:], p) + sub = append(sub, state{bs: bs, mode: smMode(bc), i: -1}) + } + + sub = append(sub, t.encode(dp+len(t.prefix), off+len(sub))...) + } + return +} + +func compile(routes []route) stateMachine { + if len(routes) == 0 { + return nil + } + t := buildTrie(routes, 0, 0) + m := t.encode(0, 0) + for i := range m { + if m[i].i == -1 { + m[i].mode = m[i].mode | smFail + } + } + return m +} diff --git a/web/router.go b/web/router.go index 3bf384d..6cf21e8 100644 --- a/web/router.go +++ b/web/router.go @@ -59,6 +59,7 @@ type router struct { lock sync.Mutex routes []route notFound Handler + machine *routeMachine } // A Pattern determines whether or not a given request matches some criteria. @@ -137,22 +138,96 @@ func httpMethod(mname string) method { return mIDK } -func (rt *router) route(c C, w http.ResponseWriter, r *http.Request) { +type routeMachine struct { + sm stateMachine + routes []route +} + +func matchRoute(route route, m method, ms *method, r *http.Request, c *C) bool { + if !route.pattern.Match(r, c, false) { + return false + } + + if route.method&m != 0 { + return true + } else { + *ms |= route.method + return false + } +} + +func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (method, bool) { m := httpMethod(r.Method) var methods method - for _, route := range rt.routes { - if !strings.HasPrefix(r.URL.Path, route.prefix) || - !route.pattern.Match(r, &c, false) { + p := r.URL.Path + + if len(rm.sm) == 0 { + return methods, false + } + var i int + for { + s := rm.sm[i] + if s.mode&smSetCursor != 0 { + p = r.URL.Path[s.i:] + i++ continue } - if route.method&m != 0 { - route.handler.ServeHTTPC(c, w, r) - return - } else if route.pattern.Match(r, &c, true) { - methods |= route.method + length := int(s.mode & smLengthMask) + match := length <= len(p) + for j := 0; match && j < length; j++ { + match = match && p[j] == s.bs[j] } + + if match { + p = p[length:] + } + + if match && s.mode&smRoute != 0 { + if matchRoute(rm.routes[s.i], m, &methods, r, c) { + rm.routes[s.i].handler.ServeHTTPC(*c, w, r) + return 0, true + } else { + i++ + } + } else if (match && s.mode&smJumpOnMatch != 0) || + (!match && s.mode&smJumpOnMatch == 0) { + + if s.mode&smFail != 0 { + return methods, false + } + i = int(s.i) + } else { + i++ + } + } + + return methods, false +} + +// Compile the list of routes into bytecode. This only needs to be done once +// after all the routes have been added, and will be called automatically for +// you (at some performance cost on the first request) if you do not call it +// explicitly. +func (rt *router) Compile() { + rt.lock.Lock() + defer rt.lock.Unlock() + sm := routeMachine{ + sm: compile(rt.routes), + routes: rt.routes, + } + rt.setMachine(&sm) +} + +func (rt *router) route(c C, w http.ResponseWriter, r *http.Request) { + if rt.machine == nil { + rt.Compile() + } + + methods, ok := rt.getMachine().route(&c, w, r) + if ok { + return } if methods == 0 { @@ -209,9 +284,7 @@ func (rt *router) handle(p Pattern, m method, h Handler) { } copy(newRoutes[i+1:], rt.routes[i:]) - // We're being a bit sloppy here: we assume that pointer assignment is - // atomic with respect to other agents that don't acquire the lock. We - // should really just give up and use sync/atomic for this. + rt.setMachine(nil) rt.routes = newRoutes } From ebc69c317325aeb2c87708ca55553d07d6aeee7d Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 3 May 2014 18:21:51 -0700 Subject: [PATCH 076/217] Router benchmarks These are based on https://github.com/cypriss/golang-mux-benchmark --- web/bench_test.go | 125 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 web/bench_test.go diff --git a/web/bench_test.go b/web/bench_test.go new file mode 100644 index 0000000..107af1b --- /dev/null +++ b/web/bench_test.go @@ -0,0 +1,125 @@ +package web + +import ( + "crypto/rand" + "encoding/base64" + mrand "math/rand" + "net/http" + "testing" + "time" +) + +func init() { + mrand.Seed(time.Now().Unix()) +} + +/* +The core benchmarks here are based on cypriss's mux benchmarks, which can be +found here: +https://github.com/cypriss/golang-mux-benchmark + +They happen to play very well into Goji's router's strengths. +*/ + +type nilRouter struct{} + +var helloWorld = []byte("Hello world!\n") + +func (_ nilRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Write(helloWorld) +} + +type nilResponse struct{} + +func (_ nilResponse) Write(buf []byte) (int, error) { + return len(buf), nil +} +func (_ nilResponse) Header() http.Header { + return nil +} +func (_ nilResponse) WriteHeader(code int) { +} + +var w nilResponse + +func addRoutes(m *Mux, prefix string) { + m.Get(prefix, nilRouter{}) + m.Post(prefix, nilRouter{}) + m.Get(prefix+"/:id", nilRouter{}) + m.Put(prefix+"/:id", nilRouter{}) + m.Delete(prefix+"/:id", nilRouter{}) +} + +func randString() string { + var buf [6]byte + rand.Reader.Read(buf[:]) + return base64.URLEncoding.EncodeToString(buf[:]) +} + +func genPrefixes(n int) []string { + p := make([]string, n) + for i := range p { + p[i] = "/" + randString() + } + return p +} + +func genRequests(prefixes []string) []*http.Request { + rs := make([]*http.Request, 5*len(prefixes)) + for i, prefix := range prefixes { + rs[5*i+0], _ = http.NewRequest("GET", prefix, nil) + rs[5*i+1], _ = http.NewRequest("POST", prefix, nil) + rs[5*i+2], _ = http.NewRequest("GET", prefix+"/foo", nil) + rs[5*i+3], _ = http.NewRequest("PUT", prefix+"/foo", nil) + rs[5*i+4], _ = http.NewRequest("DELETE", prefix+"/foo", nil) + } + return rs +} + +func permuteRequests(reqs []*http.Request) []*http.Request { + out := make([]*http.Request, len(reqs)) + perm := mrand.Perm(len(reqs)) + for i, req := range reqs { + out[perm[i]] = req + } + return out +} + +func testingMux(n int) (*Mux, []*http.Request) { + m := New() + prefixes := genPrefixes(n) + for _, prefix := range prefixes { + addRoutes(m, prefix) + } + reqs := permuteRequests(genRequests(prefixes)) + return m, reqs +} + +func BenchmarkRoute5(b *testing.B) { + m, reqs := testingMux(1) + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.ServeHTTP(w, reqs[i%len(reqs)]) + } +} +func BenchmarkRoute50(b *testing.B) { + m, reqs := testingMux(10) + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.ServeHTTP(w, reqs[i%len(reqs)]) + } +} +func BenchmarkRoute500(b *testing.B) { + m, reqs := testingMux(100) + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.ServeHTTP(w, reqs[i%len(reqs)]) + } +} +func BenchmarkRoute5000(b *testing.B) { + m, reqs := testingMux(1000) + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.ServeHTTP(w, reqs[i%len(reqs)]) + } +} From 3ea0cd7a66bc3fdc354da700fd90654fbbb33949 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 3 May 2014 18:31:06 -0700 Subject: [PATCH 077/217] Also report on allocations --- web/bench_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/bench_test.go b/web/bench_test.go index 107af1b..051fe26 100644 --- a/web/bench_test.go +++ b/web/bench_test.go @@ -98,6 +98,7 @@ func testingMux(n int) (*Mux, []*http.Request) { func BenchmarkRoute5(b *testing.B) { m, reqs := testingMux(1) b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { m.ServeHTTP(w, reqs[i%len(reqs)]) } @@ -105,6 +106,7 @@ func BenchmarkRoute5(b *testing.B) { func BenchmarkRoute50(b *testing.B) { m, reqs := testingMux(10) b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { m.ServeHTTP(w, reqs[i%len(reqs)]) } @@ -112,6 +114,7 @@ func BenchmarkRoute50(b *testing.B) { func BenchmarkRoute500(b *testing.B) { m, reqs := testingMux(100) b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { m.ServeHTTP(w, reqs[i%len(reqs)]) } @@ -119,6 +122,7 @@ func BenchmarkRoute500(b *testing.B) { func BenchmarkRoute5000(b *testing.B) { m, reqs := testingMux(1000) b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { m.ServeHTTP(w, reqs[i%len(reqs)]) } From 0861586b4e3f5f10cfbd2c5226a35a270c6b8824 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 4 May 2014 08:42:55 -0700 Subject: [PATCH 078/217] Reduce allocations in string patterns --- web/pattern.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/web/pattern.go b/web/pattern.go index 9599bfb..69f5654 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -151,7 +151,10 @@ func (s stringPattern) Prefix() string { func (s stringPattern) Match(r *http.Request, c *C, dryrun bool) bool { path := r.URL.Path - matches := make([]string, len(s.pats)) + var matches map[string]string + if !dryrun && len(s.pats) > 0 { + matches = make(map[string]string, len(s.pats)) + } for i := 0; i < len(s.pats); i++ { if !strings.HasPrefix(path, s.literals[i]) { return false @@ -167,7 +170,9 @@ func (s stringPattern) Match(r *http.Request, c *C, dryrun bool) bool { // "/:foo" would match the path "/" return false } - matches[i] = path[:m] + if !dryrun { + matches[s.pats[i]] = path[:m] + } path = path[m:] } // There's exactly one more literal than pat. @@ -185,11 +190,12 @@ func (s stringPattern) Match(r *http.Request, c *C, dryrun bool) bool { return true } - if c.URLParams == nil && len(matches) > 0 { - c.URLParams = make(map[string]string, len(matches)-1) - } - for i, match := range matches { - c.URLParams[s.pats[i]] = match + if c.URLParams == nil { + c.URLParams = matches + } else { + for k, v := range matches { + c.URLParams[k] = v + } } return true } From 33a3e80aa83961f1125bf461f42419a8d3bf2f04 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 4 May 2014 08:51:20 -0700 Subject: [PATCH 079/217] Fix routing behavior The fast routing diff introduced a regression with how method sets were calculated for routes that did not match. This fixes that behavior, as well as making routing considerably more memory-efficient (and therefore CPU-efficient too) for the case in which many routes share a prefix. --- web/router.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/web/router.go b/web/router.go index 6cf21e8..2d94d8f 100644 --- a/web/router.go +++ b/web/router.go @@ -144,16 +144,15 @@ type routeMachine struct { } func matchRoute(route route, m method, ms *method, r *http.Request, c *C) bool { - if !route.pattern.Match(r, c, false) { + if !route.pattern.Match(r, c, true) { return false } + *ms |= route.method if route.method&m != 0 { - return true - } else { - *ms |= route.method - return false + return route.pattern.Match(r, c, false) } + return false } func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (method, bool) { From 3425950f2104007f4b903d4737ad07313aca0963 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 4 May 2014 09:15:09 -0700 Subject: [PATCH 080/217] DRY up the benchmarks a bit --- web/bench_test.go | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/web/bench_test.go b/web/bench_test.go index 051fe26..6e198c3 100644 --- a/web/bench_test.go +++ b/web/bench_test.go @@ -85,45 +85,30 @@ func permuteRequests(reqs []*http.Request) []*http.Request { return out } -func testingMux(n int) (*Mux, []*http.Request) { +func benchN(b *testing.B, n int) { m := New() prefixes := genPrefixes(n) for _, prefix := range prefixes { addRoutes(m, prefix) } reqs := permuteRequests(genRequests(prefixes)) - return m, reqs -} -func BenchmarkRoute5(b *testing.B) { - m, reqs := testingMux(1) b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { m.ServeHTTP(w, reqs[i%len(reqs)]) } } + +func BenchmarkRoute5(b *testing.B) { + benchN(b, 1) +} func BenchmarkRoute50(b *testing.B) { - m, reqs := testingMux(10) - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - m.ServeHTTP(w, reqs[i%len(reqs)]) - } + benchN(b, 10) } func BenchmarkRoute500(b *testing.B) { - m, reqs := testingMux(100) - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - m.ServeHTTP(w, reqs[i%len(reqs)]) - } + benchN(b, 100) } func BenchmarkRoute5000(b *testing.B) { - m, reqs := testingMux(1000) - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - m.ServeHTTP(w, reqs[i%len(reqs)]) - } + benchN(b, 1000) } From e1ff3ade4d0ed6971f753cd7d85cf663257bb1db Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 4 May 2014 09:30:04 -0700 Subject: [PATCH 081/217] Also benchmark a single route, middleware --- web/bench_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/web/bench_test.go b/web/bench_test.go index 6e198c3..57ecf41 100644 --- a/web/bench_test.go +++ b/web/bench_test.go @@ -6,13 +6,8 @@ import ( mrand "math/rand" "net/http" "testing" - "time" ) -func init() { - mrand.Seed(time.Now().Unix()) -} - /* The core benchmarks here are based on cypriss's mux benchmarks, which can be found here: @@ -40,6 +35,13 @@ func (_ nilResponse) Header() http.Header { func (_ nilResponse) WriteHeader(code int) { } +func trivialMiddleware(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) +} + var w nilResponse func addRoutes(m *Mux, prefix string) { @@ -100,6 +102,33 @@ func benchN(b *testing.B, n int) { } } +func benchM(b *testing.B, n int) { + m := New() + m.Get("/", nilRouter{}) + for i := 0; i < n; i++ { + m.Use(trivialMiddleware) + } + r, _ := http.NewRequest("GET", "/", nil) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + m.ServeHTTP(w, r) + } +} + +func BenchmarkStatic(b *testing.B) { + m := New() + m.Get("/", nilRouter{}) + r, _ := http.NewRequest("GET", "/", nil) + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + m.ServeHTTP(w, r) + } +} + func BenchmarkRoute5(b *testing.B) { benchN(b, 1) } @@ -112,3 +141,13 @@ func BenchmarkRoute500(b *testing.B) { func BenchmarkRoute5000(b *testing.B) { benchN(b, 1000) } + +func BenchmarkMiddleware1(b *testing.B) { + benchM(b, 1) +} +func BenchmarkMiddleware10(b *testing.B) { + benchM(b, 10) +} +func BenchmarkMiddleware100(b *testing.B) { + benchM(b, 100) +} From 7f50cf60d3946726f7b91e256528840870aca48a Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 4 May 2014 09:57:01 -0700 Subject: [PATCH 082/217] Don't worry so much about helping the GC Let's just hope the GC does its job correctly and don't try to help it out. This case is probably triggered very infrequently since most people set up their middleware before they accept a single request, and it's worth about 100ns of perf on the common case for us if we get rid of the defer. --- web/middleware.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/web/middleware.go b/web/middleware.go index f9e7963..d39a428 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -82,12 +82,7 @@ func (m *mStack) findLayer(l interface{}) int { } func (m *mStack) invalidate() { - old := m.pool m.pool = make(chan *cStack, mPoolSize) - close(old) - // Bleed down the old pool so it gets GC'd - for _ = range old { - } } func (m *mStack) newStack() *cStack { @@ -134,13 +129,6 @@ func (m *mStack) release(cs *cStack) { if cs.pool != m.pool { return } - // It's possible that the pool has been invalidated (and closed) between - // the check above and now, in which case we'll start panicing, which is - // dumb. I'm not sure this is actually better than just grabbing a lock, - // but whatever. - defer func() { - recover() - }() select { case cs.pool <- cs: default: From 014650da071326a9ee2edf3c6a1b5decabd1b5cf Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 4 May 2014 10:02:04 -0700 Subject: [PATCH 083/217] Only make a best-effort to release mStacks It doesn't *actually* matter, and getting rid of the defer is worth ~80ns of perf. --- web/mux.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/web/mux.go b/web/mux.go index fcc67e8..e40eaf1 100644 --- a/web/mux.go +++ b/web/mux.go @@ -74,16 +74,14 @@ func New() *Mux { func (m *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) { stack := m.mStack.alloc() - defer m.mStack.release(stack) - stack.ServeHTTP(w, r) + m.mStack.release(stack) } // ServeHTTPC creates a context dependent request with the given Mux. Satisfies // the web.Handler interface. func (m *Mux) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { stack := m.mStack.alloc() - defer m.mStack.release(stack) - stack.ServeHTTPC(c, w, r) + m.mStack.release(stack) } From bf2f0737850f48980428c52ea4ceeb221d656b0f Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 4 May 2014 10:40:22 -0700 Subject: [PATCH 084/217] Pass a *C between middleware and router Previously, the middleware stack passed the router a C, but this was both odd semantically (a pattern which mutated the environment might see a *different* environment) and bad for perf: it cost us an allocation. Now we only pass around *C's internally. Importantly ("importantly"), this gets us down to 0 allocations for the static routing case, and one allocation (the URLParams map) for the normal routing case. --- web/middleware.go | 8 ++++++-- web/middleware_test.go | 14 ++++++++++---- web/mux.go | 2 +- web/router.go | 8 ++++---- web/router_test.go | 14 +++++++------- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/web/middleware.go b/web/middleware.go index d39a428..a8477c5 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -19,7 +19,11 @@ type mStack struct { lock sync.Mutex stack []mLayer pool chan *cStack - router Handler + router internalRouter +} + +type internalRouter interface { + route(*C, http.ResponseWriter, *http.Request) } /* @@ -93,7 +97,7 @@ func (m *mStack) newStack() *cStack { router := m.router cs.m = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - router.ServeHTTPC(cs.C, w, r) + router.route(&cs.C, w, r) }) for i := len(m.stack) - 1; i >= 0; i-- { cs.m = m.stack[i].fn(&cs.C, cs.m) diff --git a/web/middleware_test.go b/web/middleware_test.go index e783427..e52027d 100644 --- a/web/middleware_test.go +++ b/web/middleware_test.go @@ -7,14 +7,20 @@ import ( "time" ) +type iRouter func(*C, http.ResponseWriter, *http.Request) + +func (i iRouter) route(c *C, w http.ResponseWriter, r *http.Request) { + i(c, w, r) +} + func makeStack(ch chan string) *mStack { - router := func(c C, w http.ResponseWriter, r *http.Request) { + router := func(c *C, w http.ResponseWriter, r *http.Request) { ch <- "router" } return &mStack{ stack: make([]mLayer, 0), pool: make(chan *cStack, mPoolSize), - router: HandlerFunc(router), + router: iRouter(router), } } @@ -202,7 +208,7 @@ func TestInvalidation(t *testing.T) { } func TestContext(t *testing.T) { - router := func(c C, w http.ResponseWriter, r *http.Request) { + router := func(c *C, w http.ResponseWriter, r *http.Request) { if c.Env["reqID"].(int) != 2 { t.Error("Request id was not 2 :(") } @@ -210,7 +216,7 @@ func TestContext(t *testing.T) { st := mStack{ stack: make([]mLayer, 0), pool: make(chan *cStack, mPoolSize), - router: HandlerFunc(router), + router: iRouter(router), } st.Use(func(c *C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { diff --git a/web/mux.go b/web/mux.go index e40eaf1..a8e9bc6 100644 --- a/web/mux.go +++ b/web/mux.go @@ -68,7 +68,7 @@ func New() *Mux { notFound: parseHandler(http.NotFound), }, } - mux.mStack.router = HandlerFunc(mux.router.route) + mux.mStack.router = &mux.router return &mux } diff --git a/web/router.go b/web/router.go index 2d94d8f..e5ed9dd 100644 --- a/web/router.go +++ b/web/router.go @@ -219,18 +219,18 @@ func (rt *router) Compile() { rt.setMachine(&sm) } -func (rt *router) route(c C, w http.ResponseWriter, r *http.Request) { +func (rt *router) route(c *C, w http.ResponseWriter, r *http.Request) { if rt.machine == nil { rt.Compile() } - methods, ok := rt.getMachine().route(&c, w, r) + methods, ok := rt.getMachine().route(c, w, r) if ok { return } if methods == 0 { - rt.notFound.ServeHTTPC(c, w, r) + rt.notFound.ServeHTTPC(*c, w, r) return } @@ -249,7 +249,7 @@ func (rt *router) route(c C, w http.ResponseWriter, r *http.Request) { } else { c.Env[ValidMethodsKey] = methodsList } - rt.notFound.ServeHTTPC(c, w, r) + rt.notFound.ServeHTTPC(*c, w, r) } func (rt *router) handleUntyped(p interface{}, m method, h interface{}) { diff --git a/web/router_test.go b/web/router_test.go index 163ea09..442903e 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -46,7 +46,7 @@ func TestMethods(t *testing.T) { for _, method := range methods { r, _ := http.NewRequest(method, "/", nil) w := httptest.NewRecorder() - rt.route(C{}, w, r) + rt.route(&C{}, w, r) select { case val := <-ch: if val != method { @@ -119,7 +119,7 @@ func TestHandlerTypes(t *testing.T) { for route, response := range testHandlerTable { r, _ := http.NewRequest("GET", route, nil) w := httptest.NewRecorder() - rt.route(C{}, w, r) + rt.route(&C{}, w, r) select { case resp := <-ch: if resp != response { @@ -209,7 +209,7 @@ func TestRouteSelection(t *testing.T) { for counter, n = range test.results { r, _ := http.NewRequest("GET", test.key, nil) w := httptest.NewRecorder() - rt.route(C{}, w, r) + rt.route(&C{}, w, r) actual := <-ichan if n != actual { t.Errorf("Expected %q @ %d to be %d, got %d", @@ -225,7 +225,7 @@ func TestNotFound(t *testing.T) { r, _ := http.NewRequest("post", "/", nil) w := httptest.NewRecorder() - rt.route(C{}, w, r) + rt.route(&C{}, w, r) if w.Code != 404 { t.Errorf("Expected 404, got %d", w.Code) } @@ -236,7 +236,7 @@ func TestNotFound(t *testing.T) { r, _ = http.NewRequest("POST", "/", nil) w = httptest.NewRecorder() - rt.route(C{}, w, r) + rt.route(&C{}, w, r) if w.Code != http.StatusTeapot { t.Errorf("Expected a teapot, got %d", w.Code) } @@ -253,7 +253,7 @@ func TestPrefix(t *testing.T) { r, _ := http.NewRequest("GET", "/hello/world", nil) w := httptest.NewRecorder() - rt.route(C{}, w, r) + rt.route(&C{}, w, r) select { case val := <-ch: if val != "/hello/world" { @@ -302,7 +302,7 @@ func TestValidMethods(t *testing.T) { for path, eMethods := range validMethodsTable { r, _ := http.NewRequest("BOGUS", path, nil) - rt.route(C{}, httptest.NewRecorder(), r) + rt.route(&C{}, httptest.NewRecorder(), r) aMethods := <-ch if !reflect.DeepEqual(eMethods, aMethods) { t.Errorf("For %q, expected %v, got %v", path, eMethods, From 802b48519bdc3b612eb5d14ca2f45cdb2a12650b Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 4 May 2014 12:09:44 -0700 Subject: [PATCH 085/217] Rewrite perf section of README We're fast now! --- README.md | 55 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index e6105a4..69477ec 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Goji [![GoDoc](https://godoc.org/github.com/zenazn/goji?status.png)](https://godoc.org/github.com/zenazn/goji) [![Build Status](https://travis-ci.org/zenazn/goji.svg)](https://travis-ci.org/zenazn/goji) ==== -Goji is a minimalistic web framework inspired by Sinatra. +Goji is a minimalistic web framework that values composability and simplicity. Example ------- @@ -37,9 +37,10 @@ Features -------- * Compatible with `net/http` -* URL patterns (both Sinatra style `/foo/:bar` patterns and regular expressions) +* URL patterns (both Sinatra style `/foo/:bar` patterns and regular expressions, + as well as [custom patterns][pattern]) * Reconfigurable middleware stack -* Context/environment objects threaded through middleware and handlers +* Context/environment object threaded through middleware and handlers * Automatic support for [Einhorn][einhorn], systemd, and [more][bind] * [Graceful shutdown][graceful], and zero-downtime graceful reload when combined with Einhorn. @@ -50,6 +51,7 @@ Features [bind]: http://godoc.org/github.com/zenazn/goji/bind [graceful]: http://godoc.org/github.com/zenazn/goji/graceful [param]: http://godoc.org/github.com/zenazn/goji/param +[pattern]: https://godoc.org/github.com/zenazn/goji/web#Pattern Is it any good? @@ -62,10 +64,10 @@ There are [plenty][revel] of [other][gorilla] [good][pat] [Go][martini] novel, nor is it uniquely good. The primary difference between Goji and other frameworks--and the primary reason I think Goji is any good--is its philosophy: -Goji first of all attempts to be simple. It is of the Sinatra school of web -framework design, and not the Rails one. If you want me to tell you what -directory you should put your models in, or if you want built-in flash sessions, -you won't have a good time with Goji. +Goji first of all attempts to be simple. It is of the Sinatra and Flask school +of web framework design, and not the Rails/Django one. If you want me to tell +you what directory you should put your models in, or if you want built-in flash +sessions, you won't have a good time with Goji. Secondly, Goji attempts to be composable. It is fully composable with net/http, and can be used as a `http.Handler`, or can serve arbitrary `http.Handler`s. At @@ -101,28 +103,31 @@ motivations behind abandoning pat to write Goji. Is it fast? ----------- -It's not bad: in very informal tests it performed roughly in the middle of the -pack of [one set of benchmarks][bench]. For almost all applications this means -that it's fast enough that it doesn't matter. +[Yeah][bench1], [it is][bench2]. Goji is among the fastest HTTP routers out +there, and is very gentle on the garbage collector. -I have very little interest in boosting Goji's router's benchmark scores. There -is an obvious solution here--radix trees--and maybe if I get bored I'll -implement one for Goji, but I think the API guarantees and conceptual simplicity -Goji provides are more important (all routes are attempted, one after another, -until a matching route is found). Even if I choose to optimize Goji's router, -Goji's routing semantics will not change. +But that's sort of missing the point. Almost all Go routers are fast enough for +almost all purposes. In my opinion, what matters more is how simple and flexible +the routing semantics are. -Plus, Goji provides users with the ability to create their own radix trees: by -using sub-routes you create a tree of routers and match routes in more or less -the same way as a radix tree would. But, again, the real win here in my mind -isn't the performance, but the separation of concerns you get from having your -`/admin` routes and your `/profile` routes far, far away from each other. +Goji provides results indistinguishable from naively trying routes one after +another. This means that a route added before another route will be attempted +before that route as well. This is perhaps the most simple and most intuitive +interface a router can provide, and makes routes very easy to understand and +debug. -Goji's performance isn't all about the router though, it's also about allowing -net/http to perform its built-in optimizations. Perhaps uniquely in the Go web -framework ecosystem, Goji supports net/http's transparent `sendfile(2)` support. +Goji's router is also very flexible: in addition to the standard Sinatra-style +patterns and regular expression patterns, you can define [custom +patterns][pattern] to perform whatever custom matching logic you desire. Custom +patterns of course are fully compatible with the routing semantics above. -[bench]: https://github.com/cypriss/golang-mux-benchmark/ +It's easy (and quite a bit of fun!) to get carried away by microbenchmarks, but +at the end of the day you're not going to miss those extra hundred nanoseconds +on a request. What matters is that you aren't compromising on the API for a +handful of CPU cycles. + +[bench1]: https://gist.github.com/zenazn/c5c8528efe1a00634096 +[bench2]: https://github.com/zenazn/go-http-routing-benchmark Contributing From e16aa3c10c36f67ad5f1f1796fcdfe0bfd8da25e Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 4 May 2014 17:25:23 -0700 Subject: [PATCH 086/217] Split Pattern.Match in two; get rid of dryrun The "dryrun" parameter on Pattern.Match was kind of ugly and made for an exceedingly mediocre public interface. Instead, split its functionality in two: the previous "dryrun" behavior now lives in the Match method, and Patterns now actually mutate state when Run is called. The code on the backend is of course still the same (for now), but at least the interface is a little nicer. --- web/pattern.go | 18 +++++++++++++++--- web/pattern_test.go | 3 ++- web/router.go | 16 ++++++++++------ web/router_test.go | 8 ++++++-- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/web/pattern.go b/web/pattern.go index 69f5654..903f6fd 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -19,7 +19,14 @@ type regexpPattern struct { func (p regexpPattern) Prefix() string { return p.prefix } -func (p regexpPattern) Match(r *http.Request, c *C, dryrun bool) bool { +func (p regexpPattern) Match(r *http.Request, c *C) bool { + return p.match(r, c, false) +} +func (p regexpPattern) Run(r *http.Request, c *C) { + p.match(r, c, false) +} + +func (p regexpPattern) match(r *http.Request, c *C, dryrun bool) bool { matches := p.re.FindStringSubmatch(r.URL.Path) if matches == nil || len(matches) == 0 { return false @@ -148,8 +155,13 @@ type stringPattern struct { func (s stringPattern) Prefix() string { return s.literals[0] } - -func (s stringPattern) Match(r *http.Request, c *C, dryrun bool) bool { +func (s stringPattern) Match(r *http.Request, c *C) bool { + return s.match(r, c, true) +} +func (s stringPattern) Run(r *http.Request, c *C) { + s.match(r, c, false) +} +func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { path := r.URL.Path var matches map[string]string if !dryrun && len(s.pats) > 0 { diff --git a/web/pattern_test.go b/web/pattern_test.go index 2ea3c90..6b2575f 100644 --- a/web/pattern_test.go +++ b/web/pattern_test.go @@ -162,12 +162,13 @@ func TestPatterns(t *testing.T) { } func runTest(t *testing.T, p Pattern, test patternTest) { - result := p.Match(test.r, test.c, false) + result := p.Match(test.r, test.c) if result != test.match { t.Errorf("Expected match(%v, %#v) to return %v", p, test.r.URL.Path, test.match) return } + p.Run(test.r, test.c) if !reflect.DeepEqual(test.c, test.cout) { t.Errorf("Expected a context of %v, instead got %v", test.cout, diff --git a/web/router.go b/web/router.go index e5ed9dd..5c3c041 100644 --- a/web/router.go +++ b/web/router.go @@ -78,10 +78,13 @@ type Pattern interface { Prefix() string // Returns true if the request satisfies the pattern. This function is // free to examine both the request and the context to make this - // decision. After it is certain that the request matches, this function - // should mutate or create c.URLParams if necessary, unless dryrun is - // set. - Match(r *http.Request, c *C, dryrun bool) bool + // decision. Match should not modify either argument, and since it will + // potentially be called several times over the course of matching a + // request, it should be reasonably efficient. + Match(r *http.Request, c *C) bool + // Run the pattern on the request and context, modifying the context as + // necessary to bind URL parameters or other parsed state. + Run(r *http.Request, c *C) } func parsePattern(p interface{}) Pattern { @@ -144,13 +147,14 @@ type routeMachine struct { } func matchRoute(route route, m method, ms *method, r *http.Request, c *C) bool { - if !route.pattern.Match(r, c, true) { + if !route.pattern.Match(r, c) { return false } *ms |= route.method if route.method&m != 0 { - return route.pattern.Match(r, c, false) + route.pattern.Run(r, c) + return true } return false } diff --git a/web/router_test.go b/web/router_test.go index 442903e..59955cb 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -64,9 +64,11 @@ func (t testPattern) Prefix() string { return "" } -func (t testPattern) Match(r *http.Request, c *C, dryrun bool) bool { +func (t testPattern) Match(r *http.Request, c *C) bool { return true } +func (t testPattern) Run(r *http.Request, c *C) { +} var _ Pattern = testPattern{} @@ -174,9 +176,11 @@ type rsPattern struct { func (rs rsPattern) Prefix() string { return rs.prefix } -func (rs rsPattern) Match(_ *http.Request, _ *C, _ bool) bool { +func (rs rsPattern) Match(_ *http.Request, _ *C) bool { return rs.i >= *rs.counter } +func (rs rsPattern) Run(_ *http.Request, _ *C) { +} func (rs rsPattern) ServeHTTP(_ http.ResponseWriter, _ *http.Request) { rs.ichan <- rs.i From 135bdf6b4d7c03ea4c206465253e113797344b74 Mon Sep 17 00:00:00 2001 From: Anton Date: Mon, 5 May 2014 18:40:41 +0200 Subject: [PATCH 087/217] +windows support --- bind/einhorn.go | 2 ++ bind/einhorn_windows.go | 12 ++++++++++++ bind/systemd.go | 2 ++ bind/systemd_windows.go | 6 ++++++ graceful/einhorn.go | 2 ++ 5 files changed, 24 insertions(+) create mode 100644 bind/einhorn_windows.go create mode 100644 bind/systemd_windows.go diff --git a/bind/einhorn.go b/bind/einhorn.go index f756fb0..3f7635f 100644 --- a/bind/einhorn.go +++ b/bind/einhorn.go @@ -1,3 +1,5 @@ +// +build !windows + package bind import ( diff --git a/bind/einhorn_windows.go b/bind/einhorn_windows.go new file mode 100644 index 0000000..093707f --- /dev/null +++ b/bind/einhorn_windows.go @@ -0,0 +1,12 @@ +// +build windows + +package bind + +import ( + "net" +) + +func einhornInit() {} +func einhornAck() {} +func einhornBind(fd int) (net.Listener, error) { return nil, nil } +func usingEinhorn() bool { return false } diff --git a/bind/systemd.go b/bind/systemd.go index 1648bc9..e7cd8e4 100644 --- a/bind/systemd.go +++ b/bind/systemd.go @@ -1,3 +1,5 @@ +// +build !windows + package bind import ( diff --git a/bind/systemd_windows.go b/bind/systemd_windows.go new file mode 100644 index 0000000..4ad4d20 --- /dev/null +++ b/bind/systemd_windows.go @@ -0,0 +1,6 @@ +// +build windows + +package bind + +func systemdInit() {} +func usingSystemd() bool { return false } diff --git a/graceful/einhorn.go b/graceful/einhorn.go index c8e7af2..8803ad9 100644 --- a/graceful/einhorn.go +++ b/graceful/einhorn.go @@ -1,3 +1,5 @@ +// +build !windows + package graceful import ( From 8770437237cd43b77742a0ac65640958346b7b10 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 6 May 2014 14:25:11 -0700 Subject: [PATCH 088/217] Allow bind to be configured through env vars Goji now honors two environment variables: GOJI_BIND (which accepts the full "bind" flag syntax) and PORT (which is treated as a TCP port). Ref #20. --- bind/bind.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bind/bind.go b/bind/bind.go index 60aecc4..3ddf3e6 100644 --- a/bind/bind.go +++ b/bind/bind.go @@ -41,10 +41,14 @@ func init() { systemdInit() defaultBind := ":8000" - if usingEinhorn() { + if bind := os.Getenv("GOJI_BIND"); bind != "" { + defaultBind = bind + } else if usingEinhorn() { defaultBind = "einhorn@0" } else if usingSystemd() { defaultBind = "fd@3" + } else if port := os.Getenv("PORT"); port != "" { + defaultBind = ":" + port } flag.StringVar(&bind, "bind", defaultBind, `Address to bind on. If this value has a colon, as in ":8000" or From 5ac81e93fb6cf44207071a63d6af2e06688954dc Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Fri, 16 May 2014 21:01:16 -0700 Subject: [PATCH 089/217] Reset cStack's C on release Previously, we would keep the URLParams / Env associated with a cStack around until the next request flushed them. However, this might cause either of these maps to stick around for much longer than they ought to, potentially keeping references to many, many objects. Instead, clear out the saved context on every release. --- web/middleware.go | 1 + 1 file changed, 1 insertion(+) diff --git a/web/middleware.go b/web/middleware.go index a8477c5..03e04e7 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -130,6 +130,7 @@ func (m *mStack) alloc() *cStack { } func (m *mStack) release(cs *cStack) { + cs.C = C{} if cs.pool != m.pool { return } From 190bca6bc74d4cf80ad8d49eb5e8662f9494f0e4 Mon Sep 17 00:00:00 2001 From: Matt Silverlock Date: Fri, 23 May 2014 17:13:28 +0800 Subject: [PATCH 090/217] NoCache middleware - prevents proxies/clients from caching content. --- web/middleware/nocache.go | 56 ++++++++++++++++++++++++++++++++++ web/middleware/nocache_test.go | 29 ++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 web/middleware/nocache.go create mode 100644 web/middleware/nocache_test.go diff --git a/web/middleware/nocache.go b/web/middleware/nocache.go new file mode 100644 index 0000000..54ead3b --- /dev/null +++ b/web/middleware/nocache.go @@ -0,0 +1,56 @@ +package middleware + +import ( + "fmt" + "net/http" + "time" +) + +// Unix epoch time +var epoch = fmt.Sprintf("%s", time.Unix(0, 0).Format(time.RFC1123)) + +// Taken from https://github.com/mytrile/nocache +var noCacheHeaders = map[string]string{ + "Expires": epoch, + "Cache-Control": "no-cache, private, max-age=0", + "Pragma": "no-cache", + "X-Accel-Expires": "0", +} + +var etagHeaders = []string{ + "ETag", + "If-Modified-Since", + "If-Match", + "If-None-Match", + "If-Range", + "If-Unmodified-Since", +} + +// NoCache is a simple piece of middleware that sets a number of HTTP headers to prevent +// a router (or subrouter) from being cached by an upstream proxy and/or client. +// +// As per http://wiki.nginx.org/HttpProxyModule - NoCache sets: +// Expires: Thu, 01 Jan 1970 00:00:00 UTC +// Cache-Control: no-cache, private, max-age=0 +// X-Accel-Expires: 0 +// Pragma: no-cache (for HTTP/1.0 proxies/clients) +func NoCache(h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + + // Delete any ETag headers that may have been set + for _, v := range etagHeaders { + if r.Header.Get(v) != "" { + r.Header.Del(v) + } + } + + // Set our NoCache headers + for k, v := range noCacheHeaders { + w.Header().Set(k, v) + } + + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} diff --git a/web/middleware/nocache_test.go b/web/middleware/nocache_test.go new file mode 100644 index 0000000..1fb71f6 --- /dev/null +++ b/web/middleware/nocache_test.go @@ -0,0 +1,29 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/zenazn/goji/web" +) + +func TestNoCache(t *testing.T) { + + rr := httptest.NewRecorder() + s := web.New() + + s.Use(NoCache) + r, err := http.NewRequest("GET", "/", nil) + if err != nil { + t.Fatal(err) + } + + s.ServeHTTP(rr, r) + + for k, v := range noCacheHeaders { + if rr.HeaderMap[k][0] != v { + t.Errorf("%s header not set by middleware.", k) + } + } +} From 179dc70da283073aa12eac288dfb2bdc6423181e Mon Sep 17 00:00:00 2001 From: Matt Silverlock Date: Tue, 3 Jun 2014 05:49:32 +0800 Subject: [PATCH 091/217] Removed superfluous fmt.Sprintf. --- web/middleware/nocache.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/middleware/nocache.go b/web/middleware/nocache.go index 54ead3b..ae3d260 100644 --- a/web/middleware/nocache.go +++ b/web/middleware/nocache.go @@ -1,13 +1,12 @@ package middleware import ( - "fmt" "net/http" "time" ) // Unix epoch time -var epoch = fmt.Sprintf("%s", time.Unix(0, 0).Format(time.RFC1123)) +var epoch = time.Unix(0, 0).Format(time.RFC1123) // Taken from https://github.com/mytrile/nocache var noCacheHeaders = map[string]string{ From bdefa16a3ec570f695bc6227fbc441998fd4be15 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 2 Jun 2014 15:27:56 -0700 Subject: [PATCH 092/217] Increase size of per-process request ID nonce Change the per-process nonce part of the request ID from 8 characters to 10, and wrap the entire thing in a retry loop so you can never get an "unlucky" panic. I know this will "never" happen in practice, but it doesn't hurt to make sure we never, ever have any collisions, and never, ever have any runtime panics. It's also worth documenting the math ("math") I used to calculate the numbers here. --- web/middleware/request_id.go | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/web/middleware/request_id.go b/web/middleware/request_id.go index 1bbcfe2..27e2313 100644 --- a/web/middleware/request_id.go +++ b/web/middleware/request_id.go @@ -18,19 +18,38 @@ const RequestIDKey = "reqID" var prefix string var reqid uint64 +/* +A quick note on the statistics here: we're trying to calculate the chance that +two randomly generated base62 prefixes will collide. We use the formula from +http://en.wikipedia.org/wiki/Birthday_problem + +P[m, n] \approx 1 - e^{-m^2/2n} + +We ballpark an upper bound for $m$ by imagining (for whatever reason) a server +that restarts every second over 10 years, for $m = 86400 * 365 * 10 = 315360000$ + +For a $k$ character base-62 identifier, we have $n(k) = 62^k$ + +Plugging this in, we find $P[m, n(10)] \approx 5.75%$, which is good enough for +our purposes, and is surely more than anyone would ever need in practice -- a +process that is rebooted a handful of times a day for a hundred years has less +than a millionth of a percent chance of generating two colliding IDs. +*/ + func init() { hostname, err := os.Hostname() if hostname == "" || err != nil { hostname = "localhost" } var buf [12]byte - rand.Read(buf[:]) - b64 := base64.StdEncoding.EncodeToString(buf[:]) - // Strip out annoying characters. We have something like a billion to - // one chance of having enough from 12 bytes of entropy - b64 = strings.NewReplacer("+", "", "/", "").Replace(b64) + var b64 string + for len(b64) < 10 { + rand.Read(buf[:]) + b64 = base64.StdEncoding.EncodeToString(buf[:]) + b64 = strings.NewReplacer("+", "", "/", "").Replace(b64) + } - prefix = fmt.Sprintf("%s/%s", hostname, b64[0:8]) + prefix = fmt.Sprintf("%s/%s", hostname, b64[0:10]) } // RequestID is a middleware that injects a request ID into the context of each From f45ad521d425e52884933a03962b11d80b66867c Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 2 Jun 2014 23:50:16 -0700 Subject: [PATCH 093/217] Fix typo in error message --- web/router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/router.go b/web/router.go index 5c3c041..bff96d5 100644 --- a/web/router.go +++ b/web/router.go @@ -129,7 +129,7 @@ func parseHandler(h interface{}) Handler { log.Fatalf("Unknown handler type %v. Expected a web.Handler, "+ "a http.Handler, or a function with signature func(C, "+ "http.ResponseWriter, *http.Request) or "+ - "func(http.ResponseWriter, http.Request)", h) + "func(http.ResponseWriter, *http.Request)", h) } panic("log.Fatalf does not return") } From 2b7b8cb2a16413ef78f8574af1bac6df8275b6ef Mon Sep 17 00:00:00 2001 From: Mark McGranaghan Date: Sat, 7 Jun 2014 16:53:18 -0700 Subject: [PATCH 094/217] Fix typo --- web/middleware.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/middleware.go b/web/middleware.go index 03e04e7..a91b9ad 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -110,7 +110,7 @@ func (m *mStack) alloc() *cStack { // This is a little sloppy: this is only safe if this pointer // dereference is atomic. Maybe someday I'll replace it with // sync/atomic, but for now I happen to know that on all the - // architecures I care about it happens to be atomic. + // architectures I care about it happens to be atomic. p := m.pool var cs *cStack select { From 9fb476cd7dace92ff30f43783008d98ad8809ea1 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Wed, 11 Jun 2014 10:24:05 -0700 Subject: [PATCH 095/217] Update benchmark URL @julienschmidt upstreamed my patches (and did a great job of making it look amazing!), so let's link to his benchmark instead of mine. Closes #30 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 69477ec..75e005e 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ on a request. What matters is that you aren't compromising on the API for a handful of CPU cycles. [bench1]: https://gist.github.com/zenazn/c5c8528efe1a00634096 -[bench2]: https://github.com/zenazn/go-http-routing-benchmark +[bench2]: https://github.com/julienschmidt/go-http-routing-benchmark Contributing From a54c913a6a99e6079aa093b6fee82cf5865e8089 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 12 Jun 2014 04:03:32 -0400 Subject: [PATCH 096/217] Fix GC race in bind with Einhorn This fixes a race condition between package bind and the garbage collector, where if the garbage collector ran between einhornInit and einhornBind, bind would fatal with the error "dup: bad file descriptor" The core of the bug is that Go's os.File uses runtime.SetFinalizer to register a callback to close the underlying file descriptor an os.File points at when the os.File itself is being garbage collected. However, the Einhorn initialization code in bind, in the process of ensuring that every Einhorn-passed socket had CloseOnExec set on it, allocated os.File's pointing at each of these passed file descriptors, but did not keep references to these os.File's, allowing them to be garbage collected. Subsequently, if you attempted to bind one of these sockets, you'd find that it was no longer open. This is the simplest fix to the bug, which is to only allocate an os.File when we actually attempt to bind the socket. Note that there's still a race condition here if you attempt to bind the same file descriptor twice, since a GC between the two binds will likely cause the file to be collected. Fortunately, that one can be worked around by simply not allowing such silly behavior :). Another patch that makes this more clear will follow. Closes #29. --- bind/einhorn.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bind/einhorn.go b/bind/einhorn.go index 3f7635f..8c11c75 100644 --- a/bind/einhorn.go +++ b/bind/einhorn.go @@ -38,8 +38,7 @@ func einhornInit() { // Prevent einhorn's fds from leaking to our children for i := 0; i < einhornNumFds; i++ { - fd := int(einhornFd(i).Fd()) - syscall.CloseOnExec(fd) + syscall.CloseOnExec(einhornFdMap(i)) } } @@ -47,13 +46,13 @@ func usingEinhorn() bool { return einhornNumFds > 0 } -func einhornFd(n int) *os.File { +func einhornFdMap(n int) int { name := fmt.Sprintf("EINHORN_FD_%d", n) fno, err := envInt(name) if err != nil { log.Fatal(einhornErr) } - return os.NewFile(uintptr(fno), name) + return fno } func einhornBind(n int) (net.Listener, error) { @@ -64,7 +63,8 @@ func einhornBind(n int) (net.Listener, error) { return nil, fmt.Errorf(tooBigErr, n, einhornNumFds) } - f := einhornFd(n) + fno := einhornFdMap(n) + f := os.NewFile(uintptr(fno), fmt.Sprintf("einhorn@%d", n)) return net.FileListener(f) } From 6bc38f39b3caa4867106cd2c556e3244b7267779 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 12 Jun 2014 04:37:00 -0400 Subject: [PATCH 097/217] Be more strict about file closing in bind This eliminates the race condition mentioned in a54c913a by forbidding duplicate binds to the same socket (well, at least in the sense that attempting to do so will *always* result in an error instead of nondeterministically resulting in an error). --- bind/bind.go | 1 + bind/einhorn.go | 1 + 2 files changed, 2 insertions(+) diff --git a/bind/bind.go b/bind/bind.go index 3ddf3e6..b8ff22a 100644 --- a/bind/bind.go +++ b/bind/bind.go @@ -75,6 +75,7 @@ func listenTo(bind string) (net.Listener, error) { bind, err) } f := os.NewFile(uintptr(fd), bind) + defer f.Close() return net.FileListener(f) } else if strings.HasPrefix(bind, "einhorn@") { fd, err := strconv.Atoi(bind[8:]) diff --git a/bind/einhorn.go b/bind/einhorn.go index 8c11c75..c61e322 100644 --- a/bind/einhorn.go +++ b/bind/einhorn.go @@ -65,6 +65,7 @@ func einhornBind(n int) (net.Listener, error) { fno := einhornFdMap(n) f := os.NewFile(uintptr(fno), fmt.Sprintf("einhorn@%d", n)) + defer f.Close() return net.FileListener(f) } From b504b24726cda31bf72b8385552426bac826eda1 Mon Sep 17 00:00:00 2001 From: saj1th Date: Tue, 17 Jun 2014 17:02:51 +0530 Subject: [PATCH 098/217] Fix typo Typo in examples --- web/web.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/web.go b/web/web.go index 8e49a9d..9ddc9f9 100644 --- a/web/web.go +++ b/web/web.go @@ -25,7 +25,7 @@ Use your favorite HTTP verbs: var legacyFooHttpHandler http.Handler // From elsewhere m.Get("/foo", legacyFooHttpHandler) m.Post("/bar", func(w http.ResponseWriter, r *http.Request) { - w.Write("Hello world!") + w.Write([]byte("Hello world!")) }) Bind parameters using either Sinatra-like patterns or regular expressions: @@ -35,7 +35,7 @@ Bind parameters using either Sinatra-like patterns or regular expressions: }) pattern := regexp.MustCompile(`^/ip/(?P(?:\d{1,3}\.){3}\d{1,3})$`) m.Get(pattern, func(c web.C, w http.ResponseWriter, r *http.Request) { - fmt.Printf(w, "Info for IP address %s:", c.URLParams["ip"]) + fmt.Fprintf(w, "Info for IP address %s:", c.URLParams["ip"]) }) Middleware are functions that wrap http.Handlers, just like you'd use with raw From cd7aeeef8202c3b7ef9d89eb2a590fb84b2d2279 Mon Sep 17 00:00:00 2001 From: saj1th Date: Tue, 17 Jun 2014 17:45:52 +0530 Subject: [PATCH 099/217] Fixing the doc examples Fixing typos and checks --- web/web.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/web/web.go b/web/web.go index 9ddc9f9..da0b06a 100644 --- a/web/web.go +++ b/web/web.go @@ -56,7 +56,11 @@ use the Env parameter to pass data to other middleware and to the final handler: handler := func(w http.ResponseWriter, r *http.Request) { cookie, err := r.Cookie("user") if err == nil { - c.Env["user"] = cookie.Raw + //Consider using the middleware EnvInit instead of repeating the below check + if c.Env == nil { + c.Env = make(map[string]interface{}) + } + c.Env["user"] = cookie.Value } h.ServeHTTP(w, r) } @@ -64,10 +68,10 @@ use the Env parameter to pass data to other middleware and to the final handler: }) m.Get("/baz", func(c web.C, w http.ResponseWriter, r *http.Request) { - if user, ok := c.Env["user"], ok { - w.Write("Hello " + string(user)) + if user, ok := c.Env["user"].(string); ok { + w.Write([]byte("Hello " + user)) } else { - w.Write("Hello Stranger!") + w.Write([]byte("Hello Stranger!")) } }) */ From bed37a46c7706129bb9cbb0a7672868c34337767 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 17 Jun 2014 09:42:54 -0700 Subject: [PATCH 100/217] Reformat comment in docs --- web/web.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/web.go b/web/web.go index da0b06a..a69e246 100644 --- a/web/web.go +++ b/web/web.go @@ -56,7 +56,8 @@ use the Env parameter to pass data to other middleware and to the final handler: handler := func(w http.ResponseWriter, r *http.Request) { cookie, err := r.Cookie("user") if err == nil { - //Consider using the middleware EnvInit instead of repeating the below check + // Consider using the middleware EnvInit instead + // of repeating the below check if c.Env == nil { c.Env = make(map[string]interface{}) } From ea2ee5b24689c7445c452fa82ad245babb7a4607 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Wed, 18 Jun 2014 23:04:06 -0700 Subject: [PATCH 101/217] Unroll loop in web.routeMachine.route On my machine, this is worth about 10ns at 5 routes, 20ns at 50 routes, 70ns at 500 routes (a ~5% win), and 40ns at 5000. --- web/router.go | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/web/router.go b/web/router.go index bff96d5..8719e8c 100644 --- a/web/router.go +++ b/web/router.go @@ -178,13 +178,28 @@ func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (meth } length := int(s.mode & smLengthMask) - match := length <= len(p) - for j := 0; match && j < length; j++ { - match = match && p[j] == s.bs[j] - } - - if match { - p = p[length:] + match := false + if length <= len(p) { + switch length { + case 3: + if p[2] != s.bs[2] { + break + } + fallthrough + case 2: + if p[1] != s.bs[1] { + break + } + fallthrough + case 1: + if p[0] != s.bs[0] { + break + } + fallthrough + case 0: + p = p[length:] + match = true + } } if match && s.mode&smRoute != 0 { From 67b793fbc0698d8bd16d996a281634f810325f99 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Wed, 18 Jun 2014 23:33:10 -0700 Subject: [PATCH 102/217] Don't load all of rm.sm[i] For whatever reason, Go insisted on loading rm.sm[i] in several chunks, even though it could be loaded in a single 64-bit block. Instead, let's reorder our loads to minimize the amount of memory we're uselessly moving around. This gives us about a 15% perf boost in github.com/julienschmidt/go-http-routing-benchmark's BenchmarkGoji_StaticAll, and questionable benefits (i.e., not distinguishable from noise but certainly no worse) on Goji's own benchmarks. --- web/fast_router.go | 2 +- web/router.go | 31 +++++++++++++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/web/fast_router.go b/web/fast_router.go index dea67db..b6f52b1 100644 --- a/web/fast_router.go +++ b/web/fast_router.go @@ -48,8 +48,8 @@ Unfortunately, the code implementing all of this is pretty bad (both inefficient and hard to read). Maybe someday I'll come and take a second pass at it. */ type state struct { - bs [3]byte mode smMode + bs [3]byte i int32 } type stateMachine []state diff --git a/web/router.go b/web/router.go index 8719e8c..137301c 100644 --- a/web/router.go +++ b/web/router.go @@ -170,29 +170,31 @@ func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (meth var i int for { - s := rm.sm[i] - if s.mode&smSetCursor != 0 { - p = r.URL.Path[s.i:] + sm := rm.sm[i].mode + if sm&smSetCursor != 0 { + si := rm.sm[i].i + p = r.URL.Path[si:] i++ continue } - length := int(s.mode & smLengthMask) + length := int(sm & smLengthMask) match := false if length <= len(p) { + bs := rm.sm[i].bs switch length { case 3: - if p[2] != s.bs[2] { + if p[2] != bs[2] { break } fallthrough case 2: - if p[1] != s.bs[1] { + if p[1] != bs[1] { break } fallthrough case 1: - if p[0] != s.bs[0] { + if p[0] != bs[0] { break } fallthrough @@ -202,20 +204,21 @@ func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (meth } } - if match && s.mode&smRoute != 0 { - if matchRoute(rm.routes[s.i], m, &methods, r, c) { - rm.routes[s.i].handler.ServeHTTPC(*c, w, r) + if match && sm&smRoute != 0 { + si := rm.sm[i].i + if matchRoute(rm.routes[si], m, &methods, r, c) { + rm.routes[si].handler.ServeHTTPC(*c, w, r) return 0, true } else { i++ } - } else if (match && s.mode&smJumpOnMatch != 0) || - (!match && s.mode&smJumpOnMatch == 0) { + } else if (match && sm&smJumpOnMatch != 0) || + (!match && sm&smJumpOnMatch == 0) { - if s.mode&smFail != 0 { + if sm&smFail != 0 { return methods, false } - i = int(s.i) + i = int(rm.sm[i].i) } else { i++ } From 9a6729e2eab232c62592dde59966c4a238eface0 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 19 Jun 2014 00:29:55 -0700 Subject: [PATCH 103/217] Don't look up s.literals[i] twice This is worth about 3% in github.com/julienschmidt/go-http-routing-benchmark's BenchmarkGoji_GithubAll. Not a huge win, but definitely still worth it. --- web/pattern.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/web/pattern.go b/web/pattern.go index 903f6fd..0b17e64 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -168,10 +168,11 @@ func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { matches = make(map[string]string, len(s.pats)) } for i := 0; i < len(s.pats); i++ { - if !strings.HasPrefix(path, s.literals[i]) { + sli := s.literals[i] + if !strings.HasPrefix(path, sli) { return false } - path = path[len(s.literals[i]):] + path = path[len(sli):] m := strings.IndexRune(path, '/') if m == -1 { From d0d00ae437d7184fc6fc61cbf58747162c76ec6e Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 19 Jun 2014 00:40:14 -0700 Subject: [PATCH 104/217] Inline strings.IndexRune call This is worth about 4% on github.com/julienschmidt/go-http-routing-benchmark's BenchmarkGoji_GithubAll. --- web/pattern.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web/pattern.go b/web/pattern.go index 0b17e64..be4ef34 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -174,9 +174,11 @@ func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { } path = path[len(sli):] - m := strings.IndexRune(path, '/') - if m == -1 { - m = len(path) + m := 0 + for ; m < len(path); m++ { + if path[m] == '/' { + break + } } if m == 0 { // Empty strings are not matches, otherwise routes like From 6ab8e0be75df834dea116c2de9a5fb097f4518db Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 19 Jun 2014 09:50:17 -0700 Subject: [PATCH 105/217] Add Go 1.3 to Travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 7d3def7..8d7d11f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: go go: - 1.2 + - 1.3 - tip install: - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v From 1c4f21ce5f5485fca9b9ddc74dc1222c792b4c31 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 21 Jun 2014 23:13:45 -0700 Subject: [PATCH 106/217] Record the first written status, not the last If WriteHeader is called multiple times on a http.ResponseWriter, the first status is the one that is used, not the last. Fix the wrapped writer to reflect this fact. --- web/middleware/writer_proxy.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web/middleware/writer_proxy.go b/web/middleware/writer_proxy.go index bd6c097..0142403 100644 --- a/web/middleware/writer_proxy.go +++ b/web/middleware/writer_proxy.go @@ -33,9 +33,11 @@ type basicWriter struct { } func (b *basicWriter) WriteHeader(code int) { - b.code = code - b.wroteHeader = true - b.ResponseWriter.WriteHeader(code) + if !b.wroteHeader { + b.code = code + b.wroteHeader = true + b.ResponseWriter.WriteHeader(code) + } } func (b *basicWriter) Write(buf []byte) (int, error) { b.maybeWriteHeader() From 54c3a5dbde15dc0662d1451d8b80c8702c6bb289 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 21 Jun 2014 23:44:36 -0700 Subject: [PATCH 107/217] Add note about third-party libraries --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 75e005e..89a7f10 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,19 @@ handful of CPU cycles. [bench2]: https://github.com/julienschmidt/go-http-routing-benchmark +Third-Party Libraries +--------------------- + +Goji is already compatible with a great many third-party libraries that are +themselves compatible with `net/http`, however some library authors have gone +out of their way to include Goji compatibility specifically, perhaps by +integrating more tightly with Goji's `web.C` or by providing a custom pattern +type. An informal list of such libraries is maintained [on the wiki][third]; +feel free to add to it as you see fit. + +[third]: https://github.com/zenazn/goji/wiki/Third-Party-Libraries + + Contributing ------------ From aca17e0eda8b4023686b0780df56c24003e17dd7 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 23 Jun 2014 21:07:34 -0700 Subject: [PATCH 108/217] Use %+v instead of %#v when recovering panics Many common panic values, e.g. nil pointer dereferences, don't print very well under "%#v", emitting something like "runtime.errorCString{cstr:0x54b2a4}" or similar. --- web/middleware/recoverer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/middleware/recoverer.go b/web/middleware/recoverer.go index c9b7ee5..43ad648 100644 --- a/web/middleware/recoverer.go +++ b/web/middleware/recoverer.go @@ -38,7 +38,7 @@ func printPanic(reqID string, err interface{}) { if reqID != "" { cW(&buf, bBlack, "[%s] ", reqID) } - cW(&buf, bRed, "panic: %#v", err) + cW(&buf, bRed, "panic: %+v", err) log.Print(buf.String()) } From e3228e8322d760a7aad832bce70b7e11a3b2536c Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Wed, 25 Jun 2014 18:20:37 -0700 Subject: [PATCH 109/217] Change graceful to opt-in to signal handling Previously, a set of standard signals would be handled automatically via an init() function, however that made the package difficult to use in packages in which an HTTP server would only be spawned some of the times (perhaps keyed on an environment variable or flag). Now, signals must be registered manually. By default, the top-level "goji" package automatically registers signals with graceful, so this will result in no behavior changes for most people. Fixes #35. --- goji.go | 1 + graceful/einhorn.go | 5 +---- graceful/signal.go | 20 ++++++++++---------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/goji.go b/goji.go index 80d3ce9..b68b93f 100644 --- a/goji.go +++ b/goji.go @@ -59,6 +59,7 @@ func Serve() { listener := bind.Default() log.Println("Starting Goji on", listener.Addr()) + graceful.HandleSignals() bind.Ready() err := graceful.Serve(listener, http.DefaultServeMux) diff --git a/graceful/einhorn.go b/graceful/einhorn.go index 8803ad9..082d1c4 100644 --- a/graceful/einhorn.go +++ b/graceful/einhorn.go @@ -3,7 +3,6 @@ package graceful import ( - "log" "os" "strconv" "syscall" @@ -18,7 +17,5 @@ func init() { if err != nil || mpid != os.Getppid() { return } - - log.Print("graceful: Einhorn detected, adding SIGUSR2 handler") - AddSignal(syscall.SIGUSR2) + stdSignals = append(stdSignals, syscall.SIGUSR2) } diff --git a/graceful/signal.go b/graceful/signal.go index c231a80..5c2ab15 100644 --- a/graceful/signal.go +++ b/graceful/signal.go @@ -24,28 +24,28 @@ var hookLock sync.Mutex var prehooks = make([]func(), 0) var posthooks = make([]func(), 0) +var stdSignals = []os.Signal{os.Interrupt} var sigchan = make(chan os.Signal, 1) func init() { - AddSignal(os.Interrupt) go waitForSignal() } +// HandleSignals installs signal handlers for a set of standard signals. By +// default, this set only includes keyboard interrupts, however when the package +// detects that it is running under Einhorn, a SIGUSR2 handler is installed as +// well. +func HandleSignals() { + AddSignal(stdSignals...) +} + // AddSignal adds the given signal to the set of signals that trigger a graceful -// shutdown. Note that for convenience the default interrupt (SIGINT) handler is -// installed at package load time, and unless you call ResetSignals() will be -// listened for in addition to any signals you provide by calling this function. +// shutdown. func AddSignal(sig ...os.Signal) { signal.Notify(sigchan, sig...) } // ResetSignals resets the list of signals that trigger a graceful shutdown. -// Useful if, for instance, you don't want to use the default interrupt (SIGINT) -// handler. Since we necessarily install the SIGINT handler before you have a -// chance to call ResetSignals(), there will be a brief window during which the -// set of signals this package listens for will not be as you intend. Therefore, -// if you intend on using this function, we encourage you to call it as soon as -// possible. func ResetSignals() { signal.Stop(sigchan) } From 4fca47c2726f872eded42db20f2a2b68382d941e Mon Sep 17 00:00:00 2001 From: joliv Date: Mon, 30 Jun 2014 22:57:03 -0400 Subject: [PATCH 110/217] Double dashes to em dashes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 89a7f10..7fbc620 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Maybe! There are [plenty][revel] of [other][gorilla] [good][pat] [Go][martini] [web][gocraft] [frameworks][tiger] out there. Goji is by no means especially novel, nor is it uniquely good. The primary difference between Goji and other -frameworks--and the primary reason I think Goji is any good--is its philosophy: +frameworks—and the primary reason I think Goji is any good—is its philosophy: Goji first of all attempts to be simple. It is of the Sinatra and Flask school of web framework design, and not the Rails/Django one. If you want me to tell From eab7e2ddb1fa8a42a6b23c827046621d4d9629f9 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 21 Jul 2014 11:40:54 -0700 Subject: [PATCH 111/217] Make use of flag optional in bind Expose an additional function, bind.WithFlag(), which allows callers to use the previously-default "global flag" mode. This change allows the bind string parser (etc.) to be used without unwanted side effects. The behavior of the top-level "goji" package has not been changed. Fixes #47. --- bind/bind.go | 11 +++++++++++ goji.go | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/bind/bind.go b/bind/bind.go index b8ff22a..9228f5a 100644 --- a/bind/bind.go +++ b/bind/bind.go @@ -39,7 +39,18 @@ var bind string func init() { einhornInit() systemdInit() +} +// WithFlag adds a standard flag to the global flag instance that allows +// configuration of the default socket. Users who call Default() must call this +// function before flags are parsed, for example in an init() block. +// +// When selecting the default bind string, this function will examine its +// environment for hints about what port to bind to, selecting the GOJI_BIND +// environment variable, Einhorn, systemd, the PORT environment variable, and +// the port 8000, in order. In most cases, this means that the default behavior +// of the default socket will be reasonable for use in your circumstance. +func WithFlag() { defaultBind := ":8000" if bind := os.Getenv("GOJI_BIND"); bind != "" { defaultBind = bind diff --git a/goji.go b/goji.go index b68b93f..5f2d61c 100644 --- a/goji.go +++ b/goji.go @@ -44,6 +44,10 @@ import ( "github.com/zenazn/goji/graceful" ) +func init() { + bind.WithFlag() +} + // Serve starts Goji using reasonable defaults. func Serve() { if !flag.Parsed() { From 187fd0f5fa463a063cc44eebbb4c3a1dfbe40772 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 24 Jul 2014 17:42:55 -0700 Subject: [PATCH 112/217] Add compatibility for YAML in Ruby 1.8 For Bad Reasons (tm), Ruby 1.8's YAML parser expects a lot more spaces than Ruby 1.9's does. Let's humor it and give it those spaces. --- bind/einhorn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bind/einhorn.go b/bind/einhorn.go index c61e322..e695c0e 100644 --- a/bind/einhorn.go +++ b/bind/einhorn.go @@ -70,7 +70,7 @@ func einhornBind(n int) (net.Listener, error) { } // Fun story: this is actually YAML, not JSON. -const ackMsg = `{"command":"worker:ack","pid":%d}` + "\n" +const ackMsg = `{"command": "worker:ack", "pid": %d}` + "\n" func einhornAck() { if !usingEinhorn() { From 2ab51bb36071e4db7059f65deb2a981740f87656 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 24 Jul 2014 17:56:49 -0700 Subject: [PATCH 113/217] Only add log.Lmicroseconds if log.Ltime is set Also, move it out of Serve so it's possible to override. --- goji.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/goji.go b/goji.go index 5f2d61c..f6d9335 100644 --- a/goji.go +++ b/goji.go @@ -46,6 +46,9 @@ import ( func init() { bind.WithFlag() + if fl := log.Flags(); fl&log.Ltime != 0 { + log.SetFlags(fl | log.Lmicroseconds) + } } // Serve starts Goji using reasonable defaults. @@ -54,8 +57,6 @@ func Serve() { flag.Parse() } - log.SetFlags(log.Flags() | log.Lmicroseconds) - // Install our handler at the root of the standard net/http default mux. // This allows packages like expvar to continue working as expected. http.Handle("/", DefaultMux) From 8344b8a29ecef44663897b78768fd99f58def512 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 28 Jul 2014 02:00:58 -0700 Subject: [PATCH 114/217] Fix race in state machine compilation/invalidation Previously, a state machine invalidation could have raced against an in-flight routing attempt: if the invalidation occured after the routing attempt had already completed its nil-check (choosing not to compile a new state machine) but before the state machine was atomically loaded to perform routing, the routing goroutine would begin to panic from dereferencing nil. The meat of this change is that we now return the state machine that we compiled (while still holding the lock), and we only ever interact with the state machine through atomic pointer loads. --- web/atomic.go | 4 ++-- web/router.go | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/web/atomic.go b/web/atomic.go index 1bbf48e..aab483a 100644 --- a/web/atomic.go +++ b/web/atomic.go @@ -5,10 +5,10 @@ import ( "unsafe" ) -func (rt *router) getMachine() routeMachine { +func (rt *router) getMachine() *routeMachine { ptr := (*unsafe.Pointer)(unsafe.Pointer(&rt.machine)) sm := (*routeMachine)(atomic.LoadPointer(ptr)) - return *sm + return sm } func (rt *router) setMachine(m *routeMachine) { ptr := (*unsafe.Pointer)(unsafe.Pointer(&rt.machine)) diff --git a/web/router.go b/web/router.go index 137301c..cc52d96 100644 --- a/web/router.go +++ b/web/router.go @@ -231,7 +231,7 @@ func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (meth // after all the routes have been added, and will be called automatically for // you (at some performance cost on the first request) if you do not call it // explicitly. -func (rt *router) Compile() { +func (rt *router) Compile() *routeMachine { rt.lock.Lock() defer rt.lock.Unlock() sm := routeMachine{ @@ -239,14 +239,16 @@ func (rt *router) Compile() { routes: rt.routes, } rt.setMachine(&sm) + return &sm } func (rt *router) route(c *C, w http.ResponseWriter, r *http.Request) { - if rt.machine == nil { - rt.Compile() + rm := rt.getMachine() + if rm == nil { + rm = rt.Compile() } - methods, ok := rt.getMachine().route(c, w, r) + methods, ok := rm.route(c, w, r) if ok { return } From 17b9035bcd162f59e1a1247d4a755124c37052c5 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 28 Jul 2014 02:12:27 -0700 Subject: [PATCH 115/217] Add App Engine support App Engine disallows package unsafe. As a workaround for the (unsafe) RCU atomic pointer shenanigans we pull in order to avoid taking a lock in the hot routing path, let's just grab the lock. Honestly, I doubt anyone will notice anyways, especially considering the fact that App Engine is single-threaded anyways. Fixes #52. --- web/atomic.go | 2 ++ web/atomic_appengine.go | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 web/atomic_appengine.go diff --git a/web/atomic.go b/web/atomic.go index aab483a..795d8e5 100644 --- a/web/atomic.go +++ b/web/atomic.go @@ -1,3 +1,5 @@ +// +build !appengine + package web import ( diff --git a/web/atomic_appengine.go b/web/atomic_appengine.go new file mode 100644 index 0000000..027127a --- /dev/null +++ b/web/atomic_appengine.go @@ -0,0 +1,14 @@ +// +build appengine + +package web + +func (rt *router) getMachine() *routeMachine { + rt.lock.Lock() + defer rt.lock.Unlock() + return rt.machine +} + +// We always hold the lock when calling setMachine. +func (rt *router) setMachine(m *routeMachine) { + rt.machine = m +} From ab8aa1f6d8fae7c8863780f6b1b8cb37e11e1815 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 23 Aug 2014 14:34:30 -0700 Subject: [PATCH 116/217] Use sync.Pool for go1.3 and up This has the additional benefit of removing the need for a go1.3 branch. --- web/chanpool.go | 31 +++++++++++++++++++++++++++++++ web/cpool.go | 24 ++++++++++++++++++++++++ web/middleware.go | 27 +++++++-------------------- web/middleware_test.go | 4 ++-- web/mux.go | 2 +- 5 files changed, 65 insertions(+), 23 deletions(-) create mode 100644 web/chanpool.go create mode 100644 web/cpool.go diff --git a/web/chanpool.go b/web/chanpool.go new file mode 100644 index 0000000..fbe2977 --- /dev/null +++ b/web/chanpool.go @@ -0,0 +1,31 @@ +// +build !go1.3 + +package web + +// This is an alternate implementation of Go 1.3's sync.Pool. + +// Maximum size of the pool of spare middleware stacks +const cPoolSize = 32 + +type cPool chan *cStack + +func makeCPool() *cPool { + var p cPool = make(chan *cStack, cPoolSize) + return &p +} + +func (c cPool) alloc() *cStack { + select { + case cs := <-c: + return cs + default: + return nil + } +} + +func (c cPool) release(cs *cStack) { + select { + case c <- cs: + default: + } +} diff --git a/web/cpool.go b/web/cpool.go new file mode 100644 index 0000000..98aa688 --- /dev/null +++ b/web/cpool.go @@ -0,0 +1,24 @@ +// +build go1.3 + +package web + +import "sync" + +type cPool sync.Pool + +func makeCPool() *cPool { + return &cPool{} +} + +func (c *cPool) alloc() *cStack { + cs := (*sync.Pool)(c).Get() + if cs == nil { + return nil + } else { + return cs.(*cStack) + } +} + +func (c *cPool) release(cs *cStack) { + (*sync.Pool)(c).Put(cs) +} diff --git a/web/middleware.go b/web/middleware.go index a91b9ad..882cdd4 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -7,9 +7,6 @@ import ( "sync" ) -// Maximum size of the pool of spare middleware stacks -const mPoolSize = 32 - type mLayer struct { fn func(*C, http.Handler) http.Handler orig interface{} @@ -18,7 +15,7 @@ type mLayer struct { type mStack struct { lock sync.Mutex stack []mLayer - pool chan *cStack + pool *cPool router internalRouter } @@ -46,7 +43,7 @@ cStack on the floor. type cStack struct { C m http.Handler - pool chan *cStack + pool *cPool } func (s *cStack) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -86,7 +83,7 @@ func (m *mStack) findLayer(l interface{}) int { } func (m *mStack) invalidate() { - m.pool = make(chan *cStack, mPoolSize) + m.pool = makeCPool() } func (m *mStack) newStack() *cStack { @@ -112,16 +109,8 @@ func (m *mStack) alloc() *cStack { // sync/atomic, but for now I happen to know that on all the // architectures I care about it happens to be atomic. p := m.pool - var cs *cStack - select { - case cs = <-p: - // This can happen if we race against an invalidation. It's - // completely peaceful, so long as we assume we can grab a cStack before - // our stack blows out. - if cs == nil { - return m.alloc() - } - default: + cs := p.alloc() + if cs == nil { cs = m.newStack() } @@ -134,10 +123,8 @@ func (m *mStack) release(cs *cStack) { if cs.pool != m.pool { return } - select { - case cs.pool <- cs: - default: - } + cs.pool.release(cs) + cs.pool = nil } // Append the given middleware to the middleware stack. See the documentation diff --git a/web/middleware_test.go b/web/middleware_test.go index e52027d..ef36ac9 100644 --- a/web/middleware_test.go +++ b/web/middleware_test.go @@ -19,7 +19,7 @@ func makeStack(ch chan string) *mStack { } return &mStack{ stack: make([]mLayer, 0), - pool: make(chan *cStack, mPoolSize), + pool: makeCPool(), router: iRouter(router), } } @@ -215,7 +215,7 @@ func TestContext(t *testing.T) { } st := mStack{ stack: make([]mLayer, 0), - pool: make(chan *cStack, mPoolSize), + pool: makeCPool(), router: iRouter(router), } st.Use(func(c *C, h http.Handler) http.Handler { diff --git a/web/mux.go b/web/mux.go index a8e9bc6..de5a032 100644 --- a/web/mux.go +++ b/web/mux.go @@ -61,7 +61,7 @@ func New() *Mux { mux := Mux{ mStack: mStack{ stack: make([]mLayer, 0), - pool: make(chan *cStack, mPoolSize), + pool: makeCPool(), }, router: router{ routes: make([]route, 0), From 25044d60c259e7590ad677dec268f4b49bcedc67 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 23 Aug 2014 14:41:10 -0700 Subject: [PATCH 117/217] Eliminate a lock in the request hot path If you're manipulating your middleware stack concurrently with active requests you're probably doing something wrong, and it's not worth either the complexity or runtime cost to support you hitting yourself. We can probably take this principle a bit further and disallow mutating the middleware stack after any requests have been made (which will eliminate even more complexity) but that can be a project for another day. --- web/middleware.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/web/middleware.go b/web/middleware.go index 882cdd4..6ebf379 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -87,9 +87,6 @@ func (m *mStack) invalidate() { } func (m *mStack) newStack() *cStack { - m.lock.Lock() - defer m.lock.Unlock() - cs := cStack{} router := m.router @@ -104,10 +101,6 @@ func (m *mStack) newStack() *cStack { } func (m *mStack) alloc() *cStack { - // This is a little sloppy: this is only safe if this pointer - // dereference is atomic. Maybe someday I'll replace it with - // sync/atomic, but for now I happen to know that on all the - // architectures I care about it happens to be atomic. p := m.pool cs := p.alloc() if cs == nil { @@ -130,7 +123,8 @@ func (m *mStack) release(cs *cStack) { // Append the given middleware to the middleware stack. See the documentation // for type Mux for a list of valid middleware types. // -// No attempt is made to enforce the uniqueness of middlewares. +// No attempt is made to enforce the uniqueness of middlewares. It is illegal to +// call this function concurrently with active requests. func (m *mStack) Use(middleware interface{}) { m.lock.Lock() defer m.lock.Unlock() @@ -143,7 +137,8 @@ func (m *mStack) Use(middleware interface{}) { // types. Returns an error if no middleware has the name given by "before." // // No attempt is made to enforce the uniqueness of middlewares. If the insertion -// point is ambiguous, the first (outermost) one is chosen. +// point is ambiguous, the first (outermost) one is chosen. It is illegal to +// call this function concurrently with active requests. func (m *mStack) Insert(middleware, before interface{}) error { m.lock.Lock() defer m.lock.Unlock() @@ -165,7 +160,8 @@ func (m *mStack) Insert(middleware, before interface{}) error { // no such middleware can be found. // // If the name of the middleware to delete is ambiguous, the first (outermost) -// one is chosen. +// one is chosen. It is illegal to call this function concurrently with active +// requests. func (m *mStack) Abandon(middleware interface{}) error { m.lock.Lock() defer m.lock.Unlock() From 1142ca60ec2658b7fa9c899f18add5b092a45fe2 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 23 Aug 2014 15:26:14 -0700 Subject: [PATCH 118/217] Move param to https://github.com/goji/param This package was always sort of a black sheep package, and it's time it had a separate home. --- param/crazy_test.go | 56 ----- param/error_helpers.go | 25 -- param/errors.go | 112 --------- param/param.go | 60 ----- param/param_test.go | 505 ----------------------------------------- param/parse.go | 249 -------------------- param/pebkac_test.go | 58 ----- param/struct.go | 121 ---------- param/struct_test.go | 106 --------- 9 files changed, 1292 deletions(-) delete mode 100644 param/crazy_test.go delete mode 100644 param/error_helpers.go delete mode 100644 param/errors.go delete mode 100644 param/param.go delete mode 100644 param/param_test.go delete mode 100644 param/parse.go delete mode 100644 param/pebkac_test.go delete mode 100644 param/struct.go delete mode 100644 param/struct_test.go diff --git a/param/crazy_test.go b/param/crazy_test.go deleted file mode 100644 index 46538cd..0000000 --- a/param/crazy_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package param - -import ( - "net/url" - "testing" -) - -type Crazy struct { - A *Crazy - B *Crazy - Value int - Slice []int - Map map[string]Crazy -} - -func TestCrazy(t *testing.T) { - t.Parallel() - - c := Crazy{} - err := Parse(url.Values{ - "A[B][B][A][Value]": {"1"}, - "B[A][A][Slice][]": {"3", "1", "4"}, - "B[Map][hello][A][Value]": {"8"}, - "A[Value]": {"2"}, - "A[Slice][]": {"9", "1", "1"}, - "Value": {"42"}, - }, &c) - if err != nil { - t.Error("Error parsing craziness: ", err) - } - - // Exhaustively checking everything here is going to be a huge pain, so - // let's just hope for the best, pretend NPEs don't exist, and hope that - // this test covers enough stuff that it's actually useful. - assertEqual(t, "c.A.B.B.A.Value", 1, c.A.B.B.A.Value) - assertEqual(t, "c.A.Value", 2, c.A.Value) - assertEqual(t, "c.Value", 42, c.Value) - assertEqual(t, `c.B.Map["hello"].A.Value`, 8, c.B.Map["hello"].A.Value) - - assertEqual(t, "c.A.B.B.B", (*Crazy)(nil), c.A.B.B.B) - assertEqual(t, "c.A.B.A", (*Crazy)(nil), c.A.B.A) - assertEqual(t, "c.A.A", (*Crazy)(nil), c.A.A) - - if c.Slice != nil || c.Map != nil { - t.Error("Map and Slice should not be set") - } - - sl := c.B.A.A.Slice - if len(sl) != 3 || sl[0] != 3 || sl[1] != 1 || sl[2] != 4 { - t.Error("Something is wrong with c.B.A.A.Slice") - } - sl = c.A.Slice - if len(sl) != 3 || sl[0] != 9 || sl[1] != 1 || sl[2] != 1 { - t.Error("Something is wrong with c.A.Slice") - } -} diff --git a/param/error_helpers.go b/param/error_helpers.go deleted file mode 100644 index 9477d3a..0000000 --- a/param/error_helpers.go +++ /dev/null @@ -1,25 +0,0 @@ -package param - -import ( - "errors" - "fmt" - "log" -) - -// Testing log.Fatal in tests is... not a thing. Allow tests to stub it out. -var pebkacTesting bool - -const errPrefix = "param/parse: " -const yourFault = " This is a bug in your use of the param library." - -// Problem exists between keyboard and chair. This function is used in cases of -// programmer error, i.e. an inappropriate use of the param library, to -// immediately force the program to halt with a hopefully helpful error message. -func pebkac(format string, a ...interface{}) { - err := errors.New(errPrefix + fmt.Sprintf(format, a...) + yourFault) - if pebkacTesting { - panic(err) - } else { - log.Fatal(err) - } -} diff --git a/param/errors.go b/param/errors.go deleted file mode 100644 index e6b9de6..0000000 --- a/param/errors.go +++ /dev/null @@ -1,112 +0,0 @@ -package param - -import ( - "fmt" - "reflect" -) - -// TypeError is an error type returned when param has difficulty deserializing a -// parameter value. -type TypeError struct { - // The key that was in error. - Key string - // The type that was expected. - Type reflect.Type - // The underlying error produced as part of the deserialization process, - // if one exists. - Err error -} - -func (t TypeError) Error() string { - return fmt.Sprintf("param: error parsing key %q as %v: %v", t.Key, t.Type, - t.Err) -} - -// SingletonError is an error type returned when a parameter is passed multiple -// times when only a single value is expected. For example, for a struct with -// integer field "foo", "foo=1&foo=2" will return a SingletonError with key -// "foo". -type SingletonError struct { - // The key that was in error. - Key string - // The type that was expected for that key. - Type reflect.Type - // The list of values that were provided for that key. - Values []string -} - -func (s SingletonError) Error() string { - return fmt.Sprintf("param: error parsing key %q: expected single "+ - "value but was given %d: %v", s.Key, len(s.Values), s.Values) -} - -// NestingError is an error type returned when a key is nested when the target -// type does not support nesting of the given type. For example, deserializing -// the parameter key "anint[foo]" into a struct that defines an integer param -// "anint" will produce a NestingError with key "anint" and nesting "[foo]". -type NestingError struct { - // The portion of the key that was correctly parsed into a value. - Key string - // The type of the key that was invalidly nested on. - Type reflect.Type - // The portion of the key that could not be parsed due to invalid - // nesting. - Nesting string -} - -func (n NestingError) Error() string { - return fmt.Sprintf("param: error parsing key %q: invalid nesting "+ - "%q on %s key %q", n.Key+n.Nesting, n.Nesting, n.Type, n.Key) -} - -// SyntaxErrorSubtype describes what sort of syntax error was encountered. -type SyntaxErrorSubtype int - -const ( - MissingOpeningBracket SyntaxErrorSubtype = iota + 1 - MissingClosingBracket -) - -// SyntaxError is an error type returned when a key is incorrectly formatted. -type SyntaxError struct { - // The key for which there was a syntax error. - Key string - // The subtype of the syntax error, which describes what sort of error - // was encountered. - Subtype SyntaxErrorSubtype - // The part of the key (generally the suffix) that was in error. - ErrorPart string -} - -func (s SyntaxError) Error() string { - prefix := fmt.Sprintf("param: syntax error while parsing key %q: ", - s.Key) - - switch s.Subtype { - case MissingOpeningBracket: - return prefix + fmt.Sprintf("expected opening bracket, got %q", - s.ErrorPart) - case MissingClosingBracket: - return prefix + fmt.Sprintf("expected closing bracket in %q", - s.ErrorPart) - default: - panic("switch is not exhaustive!") - } -} - -// KeyError is an error type returned when an unknown field is set on a struct. -type KeyError struct { - // The full key that was in error. - FullKey string - // The key of the struct that did not have the given field. - Key string - // The type of the struct that did not have the given field. - Type reflect.Type - // The name of the field which was not present. - Field string -} - -func (k KeyError) Error() string { - return fmt.Sprintf("param: error parsing key %q: unknown field %q on "+ - "struct %q of type %v", k.FullKey, k.Field, k.Key, k.Type) -} diff --git a/param/param.go b/param/param.go deleted file mode 100644 index 685df90..0000000 --- a/param/param.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Package param deserializes parameter values into a given struct using magical -reflection ponies. Inspired by gorilla/schema, but uses Rails/jQuery style param -encoding instead of their weird dotted syntax. In particular, this package was -written with the intent of parsing the output of jQuery.param. - -This package uses struct tags to guess what names things ought to have. If a -struct value has a "param" tag defined, it will use that. If there is no "param" -tag defined, the name part of the "json" tag will be used. If that is not -defined, the name of the field itself will be used (no case transformation is -performed). - -If the name derived in this way is the string "-", param will refuse to set that -value. - -The parser is extremely strict, and will return an error if it has any -difficulty whatsoever in parsing any parameter, or if there is any kind of type -mismatch. -*/ -package param - -import ( - "net/url" - "reflect" - "strings" -) - -// Parse the given arguments into the the given pointer to a struct object. -func Parse(params url.Values, target interface{}) (err error) { - v := reflect.ValueOf(target) - - defer func() { - if r := recover(); r != nil { - var ok bool - err, ok = r.(error) - if !ok { - panic(err) - } - } - }() - - if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct { - pebkac("Target of param.Parse must be a pointer to a struct. "+ - "We instead were passed a %v", v.Type()) - } - - el := v.Elem() - t := el.Type() - cache := cacheStruct(t) - - for key, values := range params { - sk, keytail := key, "" - if i := strings.IndexRune(key, '['); i != -1 { - sk, keytail = sk[:i], sk[i:] - } - parseStructField(cache, key, sk, keytail, values, el) - } - - return nil -} diff --git a/param/param_test.go b/param/param_test.go deleted file mode 100644 index 48f5e42..0000000 --- a/param/param_test.go +++ /dev/null @@ -1,505 +0,0 @@ -package param - -import ( - "net/url" - "reflect" - "strings" - "testing" - "time" -) - -type Everything struct { - Bool bool - Int int - Uint uint - Float float64 - Map map[string]int - Slice []int - String string - Struct Sub - Time time.Time - - PBool *bool - PInt *int - PUint *uint - PFloat *float64 - PMap *map[string]int - PSlice *[]int - PString *string - PStruct *Sub - PTime *time.Time - - PPInt **int - - ABool MyBool - AInt MyInt - AUint MyUint - AFloat MyFloat - AMap MyMap - APtr MyPtr - ASlice MySlice - AString MyString -} - -type Sub struct { - A int - B int -} - -type MyBool bool -type MyInt int -type MyUint uint -type MyFloat float64 -type MyMap map[MyString]MyInt -type MyPtr *MyInt -type MySlice []MyInt -type MyString string - -var boolAnswers = map[string]bool{ - "true": true, - "false": false, - "0": false, - "1": true, - "on": true, - "": false, -} - -var testTimeString = "1996-12-19T16:39:57-08:00" -var testTime time.Time - -func init() { - testTime, _ = time.Parse(time.RFC3339, testTimeString) -} - -func singletonErrors(t *testing.T, field, valid, invalid string) { - e := Everything{} - - err := Parse(url.Values{field: {invalid}}, &e) - if err == nil { - t.Errorf("Expected error parsing %q as %s", invalid, field) - } - - err = Parse(url.Values{field + "[]": {valid}}, &e) - if err == nil { - t.Errorf("Expected error parsing nested %s", field) - } - - err = Parse(url.Values{field + "[nested]": {valid}}, &e) - if err == nil { - t.Errorf("Expected error parsing nested %s", field) - } - - err = Parse(url.Values{field: {valid, valid}}, &e) - if err == nil { - t.Errorf("Expected error passing %s twice", field) - } -} - -func TestBool(t *testing.T) { - t.Parallel() - - for val, correct := range boolAnswers { - e := Everything{} - e.Bool = !correct - err := Parse(url.Values{"Bool": {val}}, &e) - if err != nil { - t.Error("Parse error on key: ", val) - } - assertEqual(t, "e.Bool", correct, e.Bool) - } -} - -func TestBoolTyped(t *testing.T) { - t.Parallel() - - e := Everything{} - err := Parse(url.Values{"ABool": {"true"}}, &e) - if err != nil { - t.Error("Parse error for typed bool") - } - assertEqual(t, "e.ABool", MyBool(true), e.ABool) -} - -func TestBoolErrors(t *testing.T) { - t.Parallel() - singletonErrors(t, "Bool", "true", "llama") -} - -var intAnswers = map[string]int{ - "0": 0, - "9001": 9001, - "-42": -42, -} - -func TestInt(t *testing.T) { - t.Parallel() - - for val, correct := range intAnswers { - e := Everything{} - e.Int = 1 - err := Parse(url.Values{"Int": {val}}, &e) - if err != nil { - t.Error("Parse error on key: ", val) - } - assertEqual(t, "e.Int", correct, e.Int) - } -} - -func TestIntTyped(t *testing.T) { - t.Parallel() - - e := Everything{} - err := Parse(url.Values{"AInt": {"1"}}, &e) - if err != nil { - t.Error("Parse error for typed int") - } - assertEqual(t, "e.AInt", MyInt(1), e.AInt) -} - -func TestIntErrors(t *testing.T) { - t.Parallel() - singletonErrors(t, "Int", "1", "llama") - - e := Everything{} - err := Parse(url.Values{"Int": {"4.2"}}, &e) - if err == nil { - t.Error("Expected error parsing float as int") - } -} - -var uintAnswers = map[string]uint{ - "0": 0, - "9001": 9001, -} - -func TestUint(t *testing.T) { - t.Parallel() - - for val, correct := range uintAnswers { - e := Everything{} - e.Uint = 1 - err := Parse(url.Values{"Uint": {val}}, &e) - if err != nil { - t.Error("Parse error on key: ", val) - } - assertEqual(t, "e.Uint", correct, e.Uint) - } -} - -func TestUintTyped(t *testing.T) { - t.Parallel() - - e := Everything{} - err := Parse(url.Values{"AUint": {"1"}}, &e) - if err != nil { - t.Error("Parse error for typed uint") - } - assertEqual(t, "e.AUint", MyUint(1), e.AUint) -} - -func TestUintErrors(t *testing.T) { - t.Parallel() - singletonErrors(t, "Uint", "1", "llama") - - e := Everything{} - err := Parse(url.Values{"Uint": {"4.2"}}, &e) - if err == nil { - t.Error("Expected error parsing float as uint") - } - - err = Parse(url.Values{"Uint": {"-42"}}, &e) - if err == nil { - t.Error("Expected error parsing negative number as uint") - } -} - -var floatAnswers = map[string]float64{ - "0": 0, - "9001": 9001, - "-42": -42, - "9001.0": 9001.0, - "4.2": 4.2, - "-9.000001": -9.000001, -} - -func TestFloat(t *testing.T) { - t.Parallel() - - for val, correct := range floatAnswers { - e := Everything{} - e.Float = 1 - err := Parse(url.Values{"Float": {val}}, &e) - if err != nil { - t.Error("Parse error on key: ", val) - } - assertEqual(t, "e.Float", correct, e.Float) - } -} - -func TestFloatTyped(t *testing.T) { - t.Parallel() - - e := Everything{} - err := Parse(url.Values{"AFloat": {"1.0"}}, &e) - if err != nil { - t.Error("Parse error for typed float") - } - assertEqual(t, "e.AFloat", MyFloat(1.0), e.AFloat) -} - -func TestFloatErrors(t *testing.T) { - t.Parallel() - singletonErrors(t, "Float", "1.0", "llama") -} - -func TestMap(t *testing.T) { - t.Parallel() - e := Everything{} - - err := Parse(url.Values{ - "Map[one]": {"1"}, - "Map[two]": {"2"}, - "Map[three]": {"3"}, - }, &e) - if err != nil { - t.Error("Parse error in map: ", err) - } - - for k, v := range map[string]int{"one": 1, "two": 2, "three": 3} { - if mv, ok := e.Map[k]; !ok { - t.Errorf("Key %q not in map", k) - } else { - assertEqual(t, "Map["+k+"]", v, mv) - } - } -} - -func TestMapTyped(t *testing.T) { - t.Parallel() - - e := Everything{} - err := Parse(url.Values{"AMap[one]": {"1"}}, &e) - if err != nil { - t.Error("Parse error for typed map") - } - assertEqual(t, "e.AMap[one]", MyInt(1), e.AMap[MyString("one")]) -} - -func TestMapErrors(t *testing.T) { - t.Parallel() - e := Everything{} - - err := Parse(url.Values{"Map[]": {"llama"}}, &e) - if err == nil { - t.Error("expected error parsing empty map key") - } - - err = Parse(url.Values{"Map": {"llama"}}, &e) - if err == nil { - t.Error("expected error parsing map without key") - } - - err = Parse(url.Values{"Map[": {"llama"}}, &e) - if err == nil { - t.Error("expected error parsing map with malformed key") - } -} - -func testPtr(t *testing.T, key, in string, out interface{}) { - e := Everything{} - - err := Parse(url.Values{key: {in}}, &e) - if err != nil { - t.Errorf("Parse error while parsing pointer e.%s: %v", key, err) - } - fieldKey := key - if i := strings.IndexRune(fieldKey, '['); i >= 0 { - fieldKey = fieldKey[:i] - } - v := reflect.ValueOf(e).FieldByName(fieldKey) - if v.IsNil() { - t.Errorf("Expected param to allocate pointer for e.%s", key) - } else { - assertEqual(t, "*e."+key, out, v.Elem().Interface()) - } -} - -func TestPtr(t *testing.T) { - t.Parallel() - testPtr(t, "PBool", "true", true) - testPtr(t, "PInt", "2", 2) - testPtr(t, "PUint", "2", uint(2)) - testPtr(t, "PFloat", "2.0", 2.0) - testPtr(t, "PMap[llama]", "4", map[string]int{"llama": 4}) - testPtr(t, "PSlice[]", "4", []int{4}) - testPtr(t, "PString", "llama", "llama") - testPtr(t, "PStruct[B]", "2", Sub{0, 2}) - testPtr(t, "PTime", testTimeString, testTime) - - foo := 2 - testPtr(t, "PPInt", "2", &foo) -} - -func TestPtrTyped(t *testing.T) { - t.Parallel() - - e := Everything{} - err := Parse(url.Values{"APtr": {"1"}}, &e) - if err != nil { - t.Error("Parse error for typed pointer") - } - assertEqual(t, "e.APtr", MyInt(1), *e.APtr) -} - -func TestSlice(t *testing.T) { - t.Parallel() - - e := Everything{} - err := Parse(url.Values{"Slice[]": {"3", "1", "4"}}, &e) - if err != nil { - t.Error("Parse error for slice") - } - if e.Slice == nil { - t.Fatal("Expected param to allocate a slice") - } - if len(e.Slice) != 3 { - t.Fatal("Expected a slice of length 3") - } - - assertEqual(t, "e.Slice[0]", 3, e.Slice[0]) - assertEqual(t, "e.Slice[1]", 1, e.Slice[1]) - assertEqual(t, "e.Slice[2]", 4, e.Slice[2]) -} - -func TestSliceTyped(t *testing.T) { - t.Parallel() - e := Everything{} - err := Parse(url.Values{"ASlice[]": {"3", "1", "4"}}, &e) - if err != nil { - t.Error("Parse error for typed slice") - } - if e.ASlice == nil { - t.Fatal("Expected param to allocate a slice") - } - if len(e.ASlice) != 3 { - t.Fatal("Expected a slice of length 3") - } - - assertEqual(t, "e.ASlice[0]", MyInt(3), e.ASlice[0]) - assertEqual(t, "e.ASlice[1]", MyInt(1), e.ASlice[1]) - assertEqual(t, "e.ASlice[2]", MyInt(4), e.ASlice[2]) -} - -func TestSliceErrors(t *testing.T) { - t.Parallel() - e := Everything{} - err := Parse(url.Values{"Slice": {"1"}}, &e) - if err == nil { - t.Error("expected error parsing slice without key") - } - - err = Parse(url.Values{"Slice[llama]": {"1"}}, &e) - if err == nil { - t.Error("expected error parsing slice with string key") - } - - err = Parse(url.Values{"Slice[": {"1"}}, &e) - if err == nil { - t.Error("expected error parsing malformed slice key") - } -} - -var stringAnswer = "This is the world's best string" - -func TestString(t *testing.T) { - t.Parallel() - e := Everything{} - - err := Parse(url.Values{"String": {stringAnswer}}, &e) - if err != nil { - t.Error("Parse error in string: ", err) - } - - assertEqual(t, "e.String", stringAnswer, e.String) -} - -func TestStringTyped(t *testing.T) { - t.Parallel() - - e := Everything{} - err := Parse(url.Values{"AString": {"llama"}}, &e) - if err != nil { - t.Error("Parse error for typed string") - } - assertEqual(t, "e.AString", MyString("llama"), e.AString) -} - -func TestStruct(t *testing.T) { - t.Parallel() - e := Everything{} - - err := Parse(url.Values{ - "Struct[A]": {"1"}, - }, &e) - if err != nil { - t.Error("Parse error in struct: ", err) - } - assertEqual(t, "e.Struct.A", 1, e.Struct.A) - assertEqual(t, "e.Struct.B", 0, e.Struct.B) - - err = Parse(url.Values{ - "Struct[A]": {"4"}, - "Struct[B]": {"2"}, - }, &e) - if err != nil { - t.Error("Parse error in struct: ", err) - } - assertEqual(t, "e.Struct.A", 4, e.Struct.A) - assertEqual(t, "e.Struct.B", 2, e.Struct.B) -} - -func TestStructErrors(t *testing.T) { - t.Parallel() - e := Everything{} - - err := Parse(url.Values{"Struct[]": {"llama"}}, &e) - if err == nil { - t.Error("expected error parsing empty struct key") - } - - err = Parse(url.Values{"Struct": {"llama"}}, &e) - if err == nil { - t.Error("expected error parsing struct without key") - } - - err = Parse(url.Values{"Struct[": {"llama"}}, &e) - if err == nil { - t.Error("expected error parsing malformed struct key") - } - - err = Parse(url.Values{"Struct[C]": {"llama"}}, &e) - if err == nil { - t.Error("expected error parsing unknown") - } -} - -func TestTextUnmarshaler(t *testing.T) { - t.Parallel() - e := Everything{} - - err := Parse(url.Values{"Time": {testTimeString}}, &e) - if err != nil { - t.Error("parse error for TextUnmarshaler (Time): ", err) - } - assertEqual(t, "e.Time", testTime, e.Time) -} - -func TestTextUnmarshalerError(t *testing.T) { - t.Parallel() - e := Everything{} - - err := Parse(url.Values{"Time": {"llama"}}, &e) - if err == nil { - t.Error("expected error parsing llama as time") - } -} diff --git a/param/parse.go b/param/parse.go deleted file mode 100644 index b8c069d..0000000 --- a/param/parse.go +++ /dev/null @@ -1,249 +0,0 @@ -package param - -import ( - "encoding" - "fmt" - "reflect" - "strconv" - "strings" -) - -var textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() - -// Generic parse dispatcher. This function's signature is the interface of all -// parse functions. `key` is the entire key that is currently being parsed, such -// as "foo[bar][]". `keytail` is the portion of the string that the current -// parser is responsible for, for instance "[bar][]". `values` is the list of -// values assigned to this key, and `target` is where the resulting typed value -// should be Set() to. -func parse(key, keytail string, values []string, target reflect.Value) { - t := target.Type() - if reflect.PtrTo(t).Implements(textUnmarshalerType) { - parseTextUnmarshaler(key, keytail, values, target) - return - } - - switch k := target.Kind(); k { - case reflect.Bool: - parseBool(key, keytail, values, target) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - parseInt(key, keytail, values, target) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - parseUint(key, keytail, values, target) - case reflect.Float32, reflect.Float64: - parseFloat(key, keytail, values, target) - case reflect.Map: - parseMap(key, keytail, values, target) - case reflect.Ptr: - parsePtr(key, keytail, values, target) - case reflect.Slice: - parseSlice(key, keytail, values, target) - case reflect.String: - parseString(key, keytail, values, target) - case reflect.Struct: - parseStruct(key, keytail, values, target) - - default: - pebkac("unsupported object of type %v and kind %v.", - target.Type(), k) - } -} - -// We pass down both the full key ("foo[bar][]") and the part the current layer -// is responsible for making sense of ("[bar][]"). This computes the other thing -// you probably want to know, which is the path you took to get here ("foo"). -func kpath(key, keytail string) string { - l, t := len(key), len(keytail) - return key[:l-t] -} - -// Helper for validating that a value has been passed exactly once, and that the -// user is not attempting to nest on the key. -func primitive(key, keytail string, tipe reflect.Type, values []string) { - if keytail != "" { - panic(NestingError{ - Key: kpath(key, keytail), - Type: tipe, - Nesting: keytail, - }) - } - if len(values) != 1 { - panic(SingletonError{ - Key: kpath(key, keytail), - Type: tipe, - Values: values, - }) - } -} - -func keyed(tipe reflect.Type, key, keytail string) (string, string) { - if keytail[0] != '[' { - panic(SyntaxError{ - Key: kpath(key, keytail), - Subtype: MissingOpeningBracket, - ErrorPart: keytail, - }) - } - - idx := strings.IndexRune(keytail, ']') - if idx == -1 { - panic(SyntaxError{ - Key: kpath(key, keytail), - Subtype: MissingClosingBracket, - ErrorPart: keytail[1:], - }) - } - - return keytail[1:idx], keytail[idx+1:] -} - -func parseTextUnmarshaler(key, keytail string, values []string, target reflect.Value) { - primitive(key, keytail, target.Type(), values) - - tu := target.Addr().Interface().(encoding.TextUnmarshaler) - err := tu.UnmarshalText([]byte(values[0])) - if err != nil { - panic(TypeError{ - Key: kpath(key, keytail), - Type: target.Type(), - Err: err, - }) - } -} - -func parseBool(key, keytail string, values []string, target reflect.Value) { - primitive(key, keytail, target.Type(), values) - - switch values[0] { - case "true", "1", "on": - target.SetBool(true) - case "false", "0", "": - target.SetBool(false) - default: - panic(TypeError{ - Key: kpath(key, keytail), - Type: target.Type(), - }) - } -} - -func parseInt(key, keytail string, values []string, target reflect.Value) { - t := target.Type() - primitive(key, keytail, t, values) - - i, err := strconv.ParseInt(values[0], 10, t.Bits()) - if err != nil { - panic(TypeError{ - Key: kpath(key, keytail), - Type: t, - Err: err, - }) - } - target.SetInt(i) -} - -func parseUint(key, keytail string, values []string, target reflect.Value) { - t := target.Type() - primitive(key, keytail, t, values) - - i, err := strconv.ParseUint(values[0], 10, t.Bits()) - if err != nil { - panic(TypeError{ - Key: kpath(key, keytail), - Type: t, - Err: err, - }) - } - target.SetUint(i) -} - -func parseFloat(key, keytail string, values []string, target reflect.Value) { - t := target.Type() - primitive(key, keytail, t, values) - - f, err := strconv.ParseFloat(values[0], t.Bits()) - if err != nil { - panic(TypeError{ - Key: kpath(key, keytail), - Type: t, - Err: err, - }) - } - - target.SetFloat(f) -} - -func parseString(key, keytail string, values []string, target reflect.Value) { - primitive(key, keytail, target.Type(), values) - - target.SetString(values[0]) -} - -func parseSlice(key, keytail string, values []string, target reflect.Value) { - t := target.Type() - - // BUG(carl): We currently do not handle slices of nested types. If - // support is needed, the implementation probably could be fleshed out. - if keytail != "[]" { - panic(NestingError{ - Key: kpath(key, keytail), - Type: t, - Nesting: keytail, - }) - } - - slice := reflect.MakeSlice(t, len(values), len(values)) - kp := kpath(key, keytail) - for i := range values { - // We actually cheat a little bit and modify the key so we can - // generate better debugging messages later - key := fmt.Sprintf("%s[%d]", kp, i) - parse(key, "", values[i:i+1], slice.Index(i)) - } - target.Set(slice) -} - -func parseMap(key, keytail string, values []string, target reflect.Value) { - t := target.Type() - mapkey, maptail := keyed(t, key, keytail) - - // BUG(carl): We don't support any map keys except strings, although - // there's no reason we shouldn't be able to throw the value through our - // unparsing stack. - var mk reflect.Value - if t.Key().Kind() == reflect.String { - mk = reflect.ValueOf(mapkey).Convert(t.Key()) - } else { - pebkac("key for map %v isn't a string (it's a %v).", t, t.Key()) - } - - if target.IsNil() { - target.Set(reflect.MakeMap(t)) - } - - val := target.MapIndex(mk) - if !val.IsValid() || !val.CanSet() { - // It's a teensy bit annoying that the value returned by - // MapIndex isn't Set()table if the key exists. - val = reflect.New(t.Elem()).Elem() - } - parse(key, maptail, values, val) - target.SetMapIndex(mk, val) -} - -func parseStruct(key, keytail string, values []string, target reflect.Value) { - t := target.Type() - sk, skt := keyed(t, key, keytail) - cache := cacheStruct(t) - - parseStructField(cache, key, sk, skt, values, target) -} - -func parsePtr(key, keytail string, values []string, target reflect.Value) { - t := target.Type() - - if target.IsNil() { - target.Set(reflect.New(t.Elem())) - } - parse(key, keytail, values, target.Elem()) -} diff --git a/param/pebkac_test.go b/param/pebkac_test.go deleted file mode 100644 index 71d64eb..0000000 --- a/param/pebkac_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package param - -import ( - "net/url" - "strings" - "testing" -) - -type Bad struct { - Unknown interface{} -} - -type Bad2 struct { - Unknown *interface{} -} - -type Bad3 struct { - BadMap map[int]int -} - -// These tests are not parallel so we can frob pebkac behavior in an isolated -// way - -func assertPebkac(t *testing.T, err error) { - if err == nil { - t.Error("Expected PEBKAC error message") - } else if !strings.HasSuffix(err.Error(), yourFault) { - t.Errorf("Expected PEBKAC error, but got: %v", err) - } -} - -func TestBadInputs(t *testing.T) { - pebkacTesting = true - - err := Parse(url.Values{"Unknown": {"4"}}, Bad{}) - assertPebkac(t, err) - - b := &Bad{} - err = Parse(url.Values{"Unknown": {"4"}}, &b) - assertPebkac(t, err) - - pebkacTesting = false -} - -func TestBadTypes(t *testing.T) { - pebkacTesting = true - - err := Parse(url.Values{"Unknown": {"4"}}, &Bad{}) - assertPebkac(t, err) - - err = Parse(url.Values{"Unknown": {"4"}}, &Bad2{}) - assertPebkac(t, err) - - err = Parse(url.Values{"BadMap[llama]": {"4"}}, &Bad3{}) - assertPebkac(t, err) - - pebkacTesting = false -} diff --git a/param/struct.go b/param/struct.go deleted file mode 100644 index 8af3c08..0000000 --- a/param/struct.go +++ /dev/null @@ -1,121 +0,0 @@ -package param - -import ( - "reflect" - "strings" - "sync" -) - -// We decode a lot of structs (since it's the top-level thing this library -// decodes) and it takes a fair bit of work to reflect upon the struct to figure -// out what we want to do. Instead of doing this on every invocation, we cache -// metadata about each struct the first time we see it. The upshot is that we -// save some work every time. The downside is we are forced to briefly acquire -// a lock to access the cache in a thread-safe way. If this ever becomes a -// bottleneck, both the lock and the cache can be sharded or something. -type structCache map[string]cacheLine -type cacheLine struct { - offset int - parse func(string, string, []string, reflect.Value) -} - -var cacheLock sync.RWMutex -var cache = make(map[reflect.Type]structCache) - -func cacheStruct(t reflect.Type) structCache { - cacheLock.RLock() - sc, ok := cache[t] - cacheLock.RUnlock() - - if ok { - return sc - } - - // It's okay if two people build struct caches simultaneously - sc = make(structCache) - for i := 0; i < t.NumField(); i++ { - sf := t.Field(i) - // Only unexported fields have a PkgPath; we want to only cache - // exported fields. - if sf.PkgPath != "" { - continue - } - name := extractName(sf) - if name != "-" { - sc[name] = cacheLine{i, extractHandler(t, sf)} - } - } - - cacheLock.Lock() - cache[t] = sc - cacheLock.Unlock() - - return sc -} - -// Extract the name of the given struct field, looking at struct tags as -// appropriate. -func extractName(sf reflect.StructField) string { - name := sf.Tag.Get("param") - if name == "" { - name = sf.Tag.Get("json") - idx := strings.IndexRune(name, ',') - if idx >= 0 { - name = name[:idx] - } - } - if name == "" { - name = sf.Name - } - - return name -} - -func extractHandler(s reflect.Type, sf reflect.StructField) func(string, string, []string, reflect.Value) { - if reflect.PtrTo(sf.Type).Implements(textUnmarshalerType) { - return parseTextUnmarshaler - } - - switch sf.Type.Kind() { - case reflect.Bool: - return parseBool - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return parseInt - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return parseUint - case reflect.Float32, reflect.Float64: - return parseFloat - case reflect.Map: - return parseMap - case reflect.Ptr: - return parsePtr - case reflect.Slice: - return parseSlice - case reflect.String: - return parseString - case reflect.Struct: - return parseStruct - - default: - pebkac("struct %v has illegal field %q (type %v, kind %v).", - s, sf.Name, sf.Type, sf.Type.Kind()) - return nil - } -} - -// We have to parse two types of structs: ones at the top level, whose keys -// don't have square brackets around them, and nested structs, which do. -func parseStructField(cache structCache, key, sk, keytail string, values []string, target reflect.Value) { - l, ok := cache[sk] - if !ok { - panic(KeyError{ - FullKey: key, - Key: kpath(key, keytail), - Type: target.Type(), - Field: sk, - }) - } - f := target.Field(l.offset) - - l.parse(key, keytail, values, f) -} diff --git a/param/struct_test.go b/param/struct_test.go deleted file mode 100644 index ecba3e2..0000000 --- a/param/struct_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package param - -import ( - "reflect" - "testing" -) - -type Fruity struct { - A bool - B int `json:"banana"` - C uint `param:"cherry"` - D float64 `json:"durian" param:"dragonfruit"` - E int `json:"elderberry" param:"-"` - F map[string]int `json:"-" param:"fig"` - G *int `json:"grape,omitempty"` - H []int `param:"honeydew" json:"huckleberry"` - I string `foobar:"iyokan"` - J Cheesy `param:"jackfruit" cheese:"jarlsberg"` -} - -type Cheesy struct { - A int `param:"affinois"` - B int `param:"brie"` - C int `param:"camembert"` - D int `param:"delice d'argental"` -} - -type Private struct { - Public, private int -} - -var fruityType = reflect.TypeOf(Fruity{}) -var cheesyType = reflect.TypeOf(Cheesy{}) -var privateType = reflect.TypeOf(Private{}) - -var fruityNames = []string{ - "A", "banana", "cherry", "dragonfruit", "-", "fig", "grape", "honeydew", - "I", "jackfruit", -} - -var fruityCache = map[string]cacheLine{ - "A": {0, parseBool}, - "banana": {1, parseInt}, - "cherry": {2, parseUint}, - "dragonfruit": {3, parseFloat}, - "fig": {5, parseMap}, - "grape": {6, parsePtr}, - "honeydew": {7, parseSlice}, - "I": {8, parseString}, - "jackfruit": {9, parseStruct}, -} - -func assertEqual(t *testing.T, what string, e, a interface{}) { - if !reflect.DeepEqual(e, a) { - t.Errorf("Expected %s to be %v, was actually %v", what, e, a) - } -} - -func TestNames(t *testing.T) { - t.Parallel() - - for i, val := range fruityNames { - name := extractName(fruityType.Field(i)) - assertEqual(t, "tag", val, name) - } -} - -func TestCacheStruct(t *testing.T) { - t.Parallel() - - sc := cacheStruct(fruityType) - - if len(sc) != len(fruityCache) { - t.Errorf("Cache has %d keys, but expected %d", len(sc), - len(fruityCache)) - } - - for k, v := range fruityCache { - sck, ok := sc[k] - if !ok { - t.Errorf("Could not find key %q in cache", k) - continue - } - if sck.offset != v.offset { - t.Errorf("Cache for %q: expected offset %d but got %d", - k, sck.offset, v.offset) - } - // We want to compare function pointer equality, and this - // appears to be the only way - a := reflect.ValueOf(sck.parse) - b := reflect.ValueOf(v.parse) - if a.Pointer() != b.Pointer() { - t.Errorf("Parse mismatch for %q: %v, expected %v", k, a, - b) - } - } -} - -func TestPrivate(t *testing.T) { - t.Parallel() - - sc := cacheStruct(privateType) - if len(sc) != 1 { - t.Error("Expected Private{} to have one cachable field") - } -} From f226afd43574d914db2ffba662858346f2866587 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 23 Aug 2014 15:31:34 -0700 Subject: [PATCH 119/217] Fix-ups from param move --- README.md | 2 -- example/main.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 7fbc620..8c927f6 100644 --- a/README.md +++ b/README.md @@ -44,13 +44,11 @@ Features * Automatic support for [Einhorn][einhorn], systemd, and [more][bind] * [Graceful shutdown][graceful], and zero-downtime graceful reload when combined with Einhorn. -* Ruby on Rails / jQuery style [parameter parsing][param] * High in antioxidants [einhorn]: https://github.com/stripe/einhorn [bind]: http://godoc.org/github.com/zenazn/goji/bind [graceful]: http://godoc.org/github.com/zenazn/goji/graceful -[param]: http://godoc.org/github.com/zenazn/goji/param [pattern]: https://godoc.org/github.com/zenazn/goji/web#Pattern diff --git a/example/main.go b/example/main.go index 0fa57b2..d49b2af 100644 --- a/example/main.go +++ b/example/main.go @@ -14,8 +14,8 @@ import ( "regexp" "strconv" + "github.com/goji/param" "github.com/zenazn/goji" - "github.com/zenazn/goji/param" "github.com/zenazn/goji/web" ) From 8b50302744e734f5e39721de71752de2441f44aa Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 23 Aug 2014 15:55:22 -0700 Subject: [PATCH 120/217] Minor documentation touch-ups --- web/web.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/web.go b/web/web.go index a69e246..d6b6b67 100644 --- a/web/web.go +++ b/web/web.go @@ -1,5 +1,5 @@ /* -Package web is a microframework inspired by Sinatra. +Package web implements a fast and flexible middleware stack and mux. The underlying philosophy behind this package is that net/http is a very good HTTP library which is only missing a few features. If you disagree with this @@ -41,8 +41,8 @@ Bind parameters using either Sinatra-like patterns or regular expressions: Middleware are functions that wrap http.Handlers, just like you'd use with raw net/http. Middleware functions can optionally take a context parameter, which will be threaded throughout the middleware stack and to the final handler, even -if not all of these things do not support contexts. Middleware are encouraged to -use the Env parameter to pass data to other middleware and to the final handler: +if not all of these things support contexts. Middleware are encouraged to use +the Env parameter to pass data to other middleware and to the final handler: m.Use(func(h http.Handler) http.Handler { handler := func(w http.ResponseWriter, r *http.Request) { From bf3fc52add18a7c90bf2ad123fc7eaad51c97bdb Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 23 Aug 2014 15:57:24 -0700 Subject: [PATCH 121/217] Move README buttons to their own line --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8c927f6..900d2e0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ -Goji [![GoDoc](https://godoc.org/github.com/zenazn/goji?status.png)](https://godoc.org/github.com/zenazn/goji) [![Build Status](https://travis-ci.org/zenazn/goji.svg)](https://travis-ci.org/zenazn/goji) +Goji ==== +[![GoDoc](https://godoc.org/github.com/zenazn/goji/web?status.svg)](https://godoc.org/github.com/zenazn/goji/web) [![Build Status](https://travis-ci.org/zenazn/goji.svg)](https://travis-ci.org/zenazn/goji) + Goji is a minimalistic web framework that values composability and simplicity. Example From b75b7c7951d10cac0b7ef940f4813c956cf65cdc Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 23 Aug 2014 16:19:12 -0700 Subject: [PATCH 122/217] Don't use syscall package on appengine Fixes #52. --- bind/einhorn.go | 2 +- bind/einhorn_windows.go | 2 +- bind/systemd.go | 2 +- bind/systemd_windows.go | 2 +- graceful/einhorn.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bind/einhorn.go b/bind/einhorn.go index e695c0e..dc9aa38 100644 --- a/bind/einhorn.go +++ b/bind/einhorn.go @@ -1,4 +1,4 @@ -// +build !windows +// +build !windows !appengine package bind diff --git a/bind/einhorn_windows.go b/bind/einhorn_windows.go index 093707f..18cd3b3 100644 --- a/bind/einhorn_windows.go +++ b/bind/einhorn_windows.go @@ -1,4 +1,4 @@ -// +build windows +// +build windows appengine package bind diff --git a/bind/systemd.go b/bind/systemd.go index e7cd8e4..49d0013 100644 --- a/bind/systemd.go +++ b/bind/systemd.go @@ -1,4 +1,4 @@ -// +build !windows +// +build !windows !appengine package bind diff --git a/bind/systemd_windows.go b/bind/systemd_windows.go index 4ad4d20..1ac1bad 100644 --- a/bind/systemd_windows.go +++ b/bind/systemd_windows.go @@ -1,4 +1,4 @@ -// +build windows +// +build windows appengine package bind diff --git a/graceful/einhorn.go b/graceful/einhorn.go index 082d1c4..168f5b7 100644 --- a/graceful/einhorn.go +++ b/graceful/einhorn.go @@ -1,4 +1,4 @@ -// +build !windows +// +build !windows !appengine package graceful From 94ec1b113b798263913a2f3a46c6fabd307b65d0 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 23 Aug 2014 16:20:59 -0700 Subject: [PATCH 123/217] Also rename files so they aren't lies --- bind/{einhorn_windows.go => einhorn_stub.go} | 0 bind/{systemd_windows.go => systemd_stub.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename bind/{einhorn_windows.go => einhorn_stub.go} (100%) rename bind/{systemd_windows.go => systemd_stub.go} (100%) diff --git a/bind/einhorn_windows.go b/bind/einhorn_stub.go similarity index 100% rename from bind/einhorn_windows.go rename to bind/einhorn_stub.go diff --git a/bind/systemd_windows.go b/bind/systemd_stub.go similarity index 100% rename from bind/systemd_windows.go rename to bind/systemd_stub.go From 4d9fdbbbcff1941c2981c4e750f036ac3e958016 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 23 Aug 2014 17:04:35 -0700 Subject: [PATCH 124/217] Refactor WriterProxy out of web/middleware It turns out WriterProxy is pretty generally useful, especially when defining custom http loggers. Expose it in a util package so that other packages can use it. --- web/middleware/logger.go | 29 +++++---- web/middleware/writer_proxy.go | 83 ------------------------ web/util/writer_proxy.go | 115 +++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 95 deletions(-) delete mode 100644 web/middleware/writer_proxy.go create mode 100644 web/util/writer_proxy.go diff --git a/web/middleware/logger.go b/web/middleware/logger.go index ca4773b..fce822e 100644 --- a/web/middleware/logger.go +++ b/web/middleware/logger.go @@ -7,6 +7,7 @@ import ( "time" "github.com/zenazn/goji/web" + "github.com/zenazn/goji/web/util" ) // Logger is a middleware that logs the start and end of each request, along @@ -21,11 +22,14 @@ func Logger(c *web.C, h http.Handler) http.Handler { printStart(reqID, r) - lw := wrapWriter(w) + lw := util.WrapWriter(w) t1 := time.Now() h.ServeHTTP(lw, r) - lw.maybeWriteHeader() + + if lw.Status() == 0 { + lw.WriteHeader(http.StatusOK) + } t2 := time.Now() printEnd(reqID, lw, t2.Sub(t1)) @@ -49,23 +53,24 @@ func printStart(reqID string, r *http.Request) { log.Print(buf.String()) } -func printEnd(reqID string, w writerProxy, dt time.Duration) { +func printEnd(reqID string, w util.WriterProxy, dt time.Duration) { var buf bytes.Buffer if reqID != "" { cW(&buf, bBlack, "[%s] ", reqID) } buf.WriteString("Returning ") - if w.status() < 200 { - cW(&buf, bBlue, "%03d", w.status()) - } else if w.status() < 300 { - cW(&buf, bGreen, "%03d", w.status()) - } else if w.status() < 400 { - cW(&buf, bCyan, "%03d", w.status()) - } else if w.status() < 500 { - cW(&buf, bYellow, "%03d", w.status()) + status := w.Status() + if status < 200 { + cW(&buf, bBlue, "%03d", status) + } else if status < 300 { + cW(&buf, bGreen, "%03d", status) + } else if status < 400 { + cW(&buf, bCyan, "%03d", status) + } else if status < 500 { + cW(&buf, bYellow, "%03d", status) } else { - cW(&buf, bRed, "%03d", w.status()) + cW(&buf, bRed, "%03d", status) } buf.WriteString(" in ") if dt < 500*time.Millisecond { diff --git a/web/middleware/writer_proxy.go b/web/middleware/writer_proxy.go deleted file mode 100644 index 0142403..0000000 --- a/web/middleware/writer_proxy.go +++ /dev/null @@ -1,83 +0,0 @@ -package middleware - -import ( - "bufio" - "io" - "net" - "net/http" -) - -func wrapWriter(w http.ResponseWriter) writerProxy { - _, cn := w.(http.CloseNotifier) - _, fl := w.(http.Flusher) - _, hj := w.(http.Hijacker) - _, rf := w.(io.ReaderFrom) - - bw := basicWriter{ResponseWriter: w} - if cn && fl && hj && rf { - return &fancyWriter{bw} - } - return &bw -} - -type writerProxy interface { - http.ResponseWriter - maybeWriteHeader() - status() int -} - -type basicWriter struct { - http.ResponseWriter - wroteHeader bool - code int -} - -func (b *basicWriter) WriteHeader(code int) { - if !b.wroteHeader { - b.code = code - b.wroteHeader = true - b.ResponseWriter.WriteHeader(code) - } -} -func (b *basicWriter) Write(buf []byte) (int, error) { - b.maybeWriteHeader() - return b.ResponseWriter.Write(buf) -} -func (b *basicWriter) maybeWriteHeader() { - if !b.wroteHeader { - b.WriteHeader(http.StatusOK) - } -} -func (b *basicWriter) status() int { - return b.code -} -func (b *basicWriter) Unwrap() http.ResponseWriter { - return b.ResponseWriter -} - -type fancyWriter struct { - basicWriter -} - -func (f *fancyWriter) CloseNotify() <-chan bool { - cn := f.basicWriter.ResponseWriter.(http.CloseNotifier) - return cn.CloseNotify() -} -func (f *fancyWriter) Flush() { - fl := f.basicWriter.ResponseWriter.(http.Flusher) - fl.Flush() -} -func (f *fancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { - hj := f.basicWriter.ResponseWriter.(http.Hijacker) - return hj.Hijack() -} -func (f *fancyWriter) ReadFrom(r io.Reader) (int64, error) { - rf := f.basicWriter.ResponseWriter.(io.ReaderFrom) - f.basicWriter.maybeWriteHeader() - return rf.ReadFrom(r) -} - -var _ http.CloseNotifier = &fancyWriter{} -var _ http.Flusher = &fancyWriter{} -var _ http.Hijacker = &fancyWriter{} -var _ io.ReaderFrom = &fancyWriter{} diff --git a/web/util/writer_proxy.go b/web/util/writer_proxy.go new file mode 100644 index 0000000..99e7ce0 --- /dev/null +++ b/web/util/writer_proxy.go @@ -0,0 +1,115 @@ +package util + +import ( + "bufio" + "io" + "net" + "net/http" +) + +// WriterProxy is a proxy around an http.ResponseWriter that allows you to hook +// into various parts of the response process. +type WriterProxy interface { + http.ResponseWriter + // Status returns the HTTP status of the request, or 0 if one has not + // yet been sent. + Status() int + // Tee causes the response body to be written to the given io.Writer in + // addition to proxying the writes through. Only one io.Writer can be + // tee'd to at once: setting a second one will overwrite the first. + // Writes will be sent to the proxy before being written to this + // io.Writer. It is illegal for the tee'd writer to be modified + // concurrently with writes. + Tee(io.Writer) + // Unwrap returns the original proxied target. + Unwrap() http.ResponseWriter +} + +// WrapWriter wraps an http.ResponseWriter into a proxy that allows you to hook +// into various parts of the response process. +func WrapWriter(w http.ResponseWriter) WriterProxy { + _, cn := w.(http.CloseNotifier) + _, fl := w.(http.Flusher) + _, hj := w.(http.Hijacker) + _, rf := w.(io.ReaderFrom) + + bw := basicWriter{ResponseWriter: w} + if cn && fl && hj && rf { + return &fancyWriter{bw} + } + return &bw +} + +// basicWriter wraps a http.ResponseWriter that implements the minimal +// http.ResponseWriter interface. +type basicWriter struct { + http.ResponseWriter + wroteHeader bool + code int + tee io.Writer +} + +func (b *basicWriter) WriteHeader(code int) { + if !b.wroteHeader { + b.code = code + b.wroteHeader = true + b.ResponseWriter.WriteHeader(code) + } +} +func (b *basicWriter) Write(buf []byte) (int, error) { + b.WriteHeader(http.StatusOK) + n, err := b.ResponseWriter.Write(buf) + if b.tee != nil { + _, err2 := b.tee.Write(buf[:n]) + // Prefer errors generated by the proxied writer. + if err == nil { + err = err2 + } + } + return n, err +} +func (b *basicWriter) maybeWriteHeader() { + if !b.wroteHeader { + b.WriteHeader(http.StatusOK) + } +} +func (b *basicWriter) Status() int { + return b.code +} +func (b *basicWriter) Tee(w io.Writer) { + b.tee = w +} +func (b *basicWriter) Unwrap() http.ResponseWriter { + return b.ResponseWriter +} + +// fancyWriter is a writer that additionally satisfies http.CloseNotifier, +// http.Flusher, http.Hijacker, and io.ReaderFrom. It exists for the common case +// of wrapping the http.ResponseWriter that package http gives you, in order to +// make the proxied object support the full method set of the proxied object. +type fancyWriter struct { + basicWriter +} + +func (f *fancyWriter) CloseNotify() <-chan bool { + cn := f.basicWriter.ResponseWriter.(http.CloseNotifier) + return cn.CloseNotify() +} +func (f *fancyWriter) Flush() { + fl := f.basicWriter.ResponseWriter.(http.Flusher) + fl.Flush() +} +func (f *fancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hj := f.basicWriter.ResponseWriter.(http.Hijacker) + return hj.Hijack() +} +func (f *fancyWriter) ReadFrom(r io.Reader) (int64, error) { + rf := f.basicWriter.ResponseWriter.(io.ReaderFrom) + f.basicWriter.maybeWriteHeader() + return rf.ReadFrom(r) +} + +var _ http.CloseNotifier = &fancyWriter{} +var _ http.Flusher = &fancyWriter{} +var _ http.Hijacker = &fancyWriter{} +var _ io.ReaderFrom = &fancyWriter{} From 28b92067918cb6c1cec4a4322bb9aba3c8bb79cf Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 23 Aug 2014 20:01:04 -0700 Subject: [PATCH 125/217] golint --- web/cpool.go | 3 +-- web/router.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/web/cpool.go b/web/cpool.go index 98aa688..59f8764 100644 --- a/web/cpool.go +++ b/web/cpool.go @@ -14,9 +14,8 @@ func (c *cPool) alloc() *cStack { cs := (*sync.Pool)(c).Get() if cs == nil { return nil - } else { - return cs.(*cStack) } + return cs.(*cStack) } func (c *cPool) release(cs *cStack) { diff --git a/web/router.go b/web/router.go index cc52d96..cf93d93 100644 --- a/web/router.go +++ b/web/router.go @@ -209,9 +209,8 @@ func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (meth if matchRoute(rm.routes[si], m, &methods, r, c) { rm.routes[si].handler.ServeHTTPC(*c, w, r) return 0, true - } else { - i++ } + i++ } else if (match && sm&smJumpOnMatch != 0) || (!match && sm&smJumpOnMatch == 0) { From 91e2de24198528336af849af5018d25ad99f58f6 Mon Sep 17 00:00:00 2001 From: wangjohn Date: Mon, 25 Aug 2014 22:20:42 -0700 Subject: [PATCH 126/217] Fixing typo in example README. --- example/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/README.md b/example/README.md index acb84d6..8d33103 100644 --- a/example/README.md +++ b/example/README.md @@ -5,6 +5,6 @@ Gritter is an example application built using Goji, where people who have nothing better to do can post short 140-character "greets." A good place to start is with `main.go`, which contains a well-commented -walthrough of Goji's features. Gritter uses a couple custom middlewares, which +walkthrough of Goji's features. Gritter uses a couple custom middlewares, which have been arbitrarily placed in `middleware.go`. Finally some uninteresting "database models" live in `models.go`. From e101f753417e0b0a0ce8a0af8a9a443df5339aff Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 26 Aug 2014 10:20:26 -0700 Subject: [PATCH 127/217] Boolean logic is hard, guys :( The result of this was that builds were broken on windows and app engine. Fixes #59. --- bind/einhorn.go | 2 +- bind/systemd.go | 2 +- graceful/einhorn.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bind/einhorn.go b/bind/einhorn.go index dc9aa38..d18b694 100644 --- a/bind/einhorn.go +++ b/bind/einhorn.go @@ -1,4 +1,4 @@ -// +build !windows !appengine +// +build !windows,!appengine package bind diff --git a/bind/systemd.go b/bind/systemd.go index 49d0013..292f509 100644 --- a/bind/systemd.go +++ b/bind/systemd.go @@ -1,4 +1,4 @@ -// +build !windows !appengine +// +build !windows,!appengine package bind diff --git a/graceful/einhorn.go b/graceful/einhorn.go index 168f5b7..ad78d1c 100644 --- a/graceful/einhorn.go +++ b/graceful/einhorn.go @@ -1,4 +1,4 @@ -// +build !windows !appengine +// +build !windows,!appengine package graceful From e5a18d6241fb3d7bee5c2ad94035b0e25def2e24 Mon Sep 17 00:00:00 2001 From: "R. Peres" Date: Sat, 30 Aug 2014 00:55:53 +0100 Subject: [PATCH 128/217] Update web.go --- web/web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/web.go b/web/web.go index d6b6b67..480f670 100644 --- a/web/web.go +++ b/web/web.go @@ -102,7 +102,7 @@ type C struct { // A free-form environment, similar to Rack or PEP 333's environments. // Middleware layers are encouraged to pass data to downstream layers // and other handlers using this map, and are even more strongly - // encouraged to document and maybe namespace they keys they use. + // encouraged to document and maybe namespace the keys they use. Env map[string]interface{} } From 0e21fc92a43e2b07168deb9e0ecdef2acacd7a38 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 31 Aug 2014 19:54:01 -0700 Subject: [PATCH 129/217] Document package web/util --- web/util/util.go | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 web/util/util.go diff --git a/web/util/util.go b/web/util/util.go new file mode 100644 index 0000000..aa9dda8 --- /dev/null +++ b/web/util/util.go @@ -0,0 +1,3 @@ +// Package util contains various functions that are helpful when writing http +// middleware. +package util From 8b5a2b91620538a67dd8adf5186895fed1a7ae01 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 31 Aug 2014 19:56:16 -0700 Subject: [PATCH 130/217] Support Tee() in the ReadFrom case as well --- web/util/writer_proxy.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/util/writer_proxy.go b/web/util/writer_proxy.go index 99e7ce0..3ba08de 100644 --- a/web/util/writer_proxy.go +++ b/web/util/writer_proxy.go @@ -104,6 +104,9 @@ func (f *fancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { return hj.Hijack() } func (f *fancyWriter) ReadFrom(r io.Reader) (int64, error) { + if f.basicWriter.tee != nil { + return io.Copy(&f.basicWriter, r) + } rf := f.basicWriter.ResponseWriter.(io.ReaderFrom) f.basicWriter.maybeWriteHeader() return rf.ReadFrom(r) From 90d355d3e1693435a6300da7d4e191126aced78e Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 1 Sep 2014 16:06:19 -0700 Subject: [PATCH 131/217] Use net/http.Server.SetKeepAlivesEnabled in Go 1.3 This feature can be used in place of the pile of hacks in middleware.go, and doesn't involve awkwardly shimming out a http.ResponseWriter. Sounds like a win-win! --- example/main.go | 18 ++++++++++++++++ graceful/graceful.go | 33 ----------------------------- graceful/middleware.go | 2 ++ graceful/middleware13.go | 12 +++++++++++ graceful/middleware_test.go | 2 ++ graceful/net.go | 3 +++ graceful/serve.go | 41 ++++++++++++++++++++++++++++++++++++ graceful/serve13.go | 42 +++++++++++++++++++++++++++++++++++++ 8 files changed, 120 insertions(+), 33 deletions(-) create mode 100644 graceful/middleware13.go create mode 100644 graceful/serve.go create mode 100644 graceful/serve13.go diff --git a/example/main.go b/example/main.go index d49b2af..9315af6 100644 --- a/example/main.go +++ b/example/main.go @@ -13,6 +13,7 @@ import ( "net/http" "regexp" "strconv" + "time" "github.com/goji/param" "github.com/zenazn/goji" @@ -58,6 +59,9 @@ func main() { // Use a custom 404 handler goji.NotFound(NotFound) + // Sometimes requests take a long time. + goji.Get("/waitforit", WaitForIt) + // Call Serve() at the bottom of your main() function, and it'll take // care of everything else for you, including binding to a socket (with // automatic support for systemd and Einhorn) and supporting graceful @@ -135,6 +139,20 @@ func GetGreet(c web.C, w http.ResponseWriter, r *http.Request) { greet.Write(w) } +// WaitForIt is a particularly slow handler (GET "/waitforit"). Try loading this +// endpoint and initiating a graceful shutdown (Ctrl-C) or Einhorn reload. The +// old server will stop accepting new connections and will attempt to kill +// outstanding idle (keep-alive) connections, but will patiently stick around +// for this endpoint to finish. How kind of it! +func WaitForIt(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "This is going to be legend... (wait for it)\n") + if fl, ok := w.(http.Flusher); ok { + fl.Flush() + } + time.Sleep(15 * time.Second) + io.WriteString(w, "...dary! Legendary!\n") +} + // AdminRoot is root (GET "/admin/root"). Much secret. Very administrate. Wow. func AdminRoot(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "Gritter\n======\n\nSuper secret admin page!\n") diff --git a/graceful/graceful.go b/graceful/graceful.go index 13537e7..8958126 100644 --- a/graceful/graceful.go +++ b/graceful/graceful.go @@ -22,7 +22,6 @@ import ( "crypto/tls" "net" "net/http" - "time" ) /* @@ -41,38 +40,6 @@ ListenAndServe and ListenAndServeTLS here more-or-less verbatim. "Oh well!" // implementations of its methods. type Server http.Server -func (srv *Server) Serve(l net.Listener) (err error) { - go func() { - <-kill - l.Close() - }() - l = WrapListener(l) - - // Spawn a shadow http.Server to do the actual servering. We do this - // because we need to sketch on some of the parameters you passed in, - // and it's nice to keep our sketching to ourselves. - shadow := *(*http.Server)(srv) - - if shadow.ReadTimeout == 0 { - shadow.ReadTimeout = forever - } - shadow.Handler = Middleware(shadow.Handler) - - err = shadow.Serve(l) - - // We expect an error when we close the listener, so we indiscriminately - // swallow Serve errors when we're in a shutdown state. - select { - case <-kill: - return nil - default: - return err - } -} - -// About 200 years, also known as "forever" -const forever time.Duration = 200 * 365 * 24 * time.Hour - func (srv *Server) ListenAndServe() error { addr := srv.Addr if addr == "" { diff --git a/graceful/middleware.go b/graceful/middleware.go index 3e17620..f169891 100644 --- a/graceful/middleware.go +++ b/graceful/middleware.go @@ -1,3 +1,5 @@ +// +build !go1.3 + package graceful import ( diff --git a/graceful/middleware13.go b/graceful/middleware13.go new file mode 100644 index 0000000..b1a7e2d --- /dev/null +++ b/graceful/middleware13.go @@ -0,0 +1,12 @@ +// +build go1.3 + +package graceful + +import "net/http" + +// Middleware is a stub that does nothing. When used with versions of Go before +// Go 1.3, it provides functionality similar to net/http.Server's +// SetKeepAlivesEnabled. +func Middleware(h http.Handler) http.Handler { + return h +} diff --git a/graceful/middleware_test.go b/graceful/middleware_test.go index ecec606..15b4fcc 100644 --- a/graceful/middleware_test.go +++ b/graceful/middleware_test.go @@ -1,3 +1,5 @@ +// +build !go1.3 + package graceful import ( diff --git a/graceful/net.go b/graceful/net.go index 5573796..b3f124a 100644 --- a/graceful/net.go +++ b/graceful/net.go @@ -162,6 +162,9 @@ func (c *conn) Read(b []byte) (n int, err error) { if c.state == csWaiting { c.state = csWorking + } else if c.state == csDead { + n = 0 + err = io.EOF } }() diff --git a/graceful/serve.go b/graceful/serve.go new file mode 100644 index 0000000..8746075 --- /dev/null +++ b/graceful/serve.go @@ -0,0 +1,41 @@ +// +build !go1.3 + +package graceful + +import ( + "net" + "net/http" + "time" +) + +// About 200 years, also known as "forever" +const forever time.Duration = 200 * 365 * 24 * time.Hour + +func (srv *Server) Serve(l net.Listener) error { + go func() { + <-kill + l.Close() + }() + l = WrapListener(l) + + // Spawn a shadow http.Server to do the actual servering. We do this + // because we need to sketch on some of the parameters you passed in, + // and it's nice to keep our sketching to ourselves. + shadow := *(*http.Server)(srv) + + if shadow.ReadTimeout == 0 { + shadow.ReadTimeout = forever + } + shadow.Handler = Middleware(shadow.Handler) + + err := shadow.Serve(l) + + // We expect an error when we close the listener, so we indiscriminately + // swallow Serve errors when we're in a shutdown state. + select { + case <-kill: + return nil + default: + return err + } +} diff --git a/graceful/serve13.go b/graceful/serve13.go new file mode 100644 index 0000000..41e9f52 --- /dev/null +++ b/graceful/serve13.go @@ -0,0 +1,42 @@ +// +build go1.3 + +package graceful + +import ( + "net" + "net/http" + "time" +) + +// About 200 years, also known as "forever" +const forever time.Duration = 200 * 365 * 24 * time.Hour + +func (srv *Server) Serve(l net.Listener) error { + l = WrapListener(l) + + // Spawn a shadow http.Server to do the actual servering. We do this + // because we need to sketch on some of the parameters you passed in, + // and it's nice to keep our sketching to ourselves. + shadow := *(*http.Server)(srv) + + if shadow.ReadTimeout == 0 { + shadow.ReadTimeout = forever + } + + go func() { + <-kill + shadow.SetKeepAlivesEnabled(false) + l.Close() + }() + + err := shadow.Serve(l) + + // We expect an error when we close the listener, so we indiscriminately + // swallow Serve errors when we're in a shutdown state. + select { + case <-kill: + return nil + default: + return err + } +} From 3ed4dab2289fc04f5b4f95af9edc43d14089a64b Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Wed, 1 Oct 2014 10:45:45 -0700 Subject: [PATCH 132/217] Better top-level AppEngine support --- goji.go | 41 ----------------------------------------- serve.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ serve_appengine.go | 13 +++++++++++++ 3 files changed, 57 insertions(+), 41 deletions(-) create mode 100644 serve.go create mode 100644 serve_appengine.go diff --git a/goji.go b/goji.go index f6d9335..ab278cd 100644 --- a/goji.go +++ b/goji.go @@ -34,44 +34,3 @@ methods of using this library are equally well supported. Goji requires Go 1.2 or newer. */ package goji - -import ( - "flag" - "log" - "net/http" - - "github.com/zenazn/goji/bind" - "github.com/zenazn/goji/graceful" -) - -func init() { - bind.WithFlag() - if fl := log.Flags(); fl&log.Ltime != 0 { - log.SetFlags(fl | log.Lmicroseconds) - } -} - -// Serve starts Goji using reasonable defaults. -func Serve() { - if !flag.Parsed() { - flag.Parse() - } - - // Install our handler at the root of the standard net/http default mux. - // This allows packages like expvar to continue working as expected. - http.Handle("/", DefaultMux) - - listener := bind.Default() - log.Println("Starting Goji on", listener.Addr()) - - graceful.HandleSignals() - bind.Ready() - - err := graceful.Serve(listener, http.DefaultServeMux) - - if err != nil { - log.Fatal(err) - } - - graceful.Wait() -} diff --git a/serve.go b/serve.go new file mode 100644 index 0000000..0a930c3 --- /dev/null +++ b/serve.go @@ -0,0 +1,44 @@ +// +build !appengine + +package goji + +import ( + "flag" + "log" + "net/http" + + "github.com/zenazn/goji/bind" + "github.com/zenazn/goji/graceful" +) + +func init() { + bind.WithFlag() + if fl := log.Flags(); fl&log.Ltime != 0 { + log.SetFlags(fl | log.Lmicroseconds) + } +} + +// Serve starts Goji using reasonable defaults. +func Serve() { + if !flag.Parsed() { + flag.Parse() + } + + // Install our handler at the root of the standard net/http default mux. + // This allows packages like expvar to continue working as expected. + http.Handle("/", DefaultMux) + + listener := bind.Default() + log.Println("Starting Goji on", listener.Addr()) + + graceful.HandleSignals() + bind.Ready() + + err := graceful.Serve(listener, http.DefaultServeMux) + + if err != nil { + log.Fatal(err) + } + + graceful.Wait() +} diff --git a/serve_appengine.go b/serve_appengine.go new file mode 100644 index 0000000..edf9a2d --- /dev/null +++ b/serve_appengine.go @@ -0,0 +1,13 @@ +// +build appengine + +package goji + +import "net/http" + +// Serve starts Goji using reasonable defaults. +func Serve() { + // Install our handler at the root of the standard net/http default mux. + // This is required for App Engine, and also allows packages like expvar + // to continue working as expected. + http.Handle("/", DefaultMux) +} From 22c4c96f3bf62ef5ae2074c115c99da857452b75 Mon Sep 17 00:00:00 2001 From: Marcus Redivo Date: Fri, 3 Oct 2014 14:12:14 -0700 Subject: [PATCH 133/217] Add BytesWritten() method to WriterProxy. This makes it possible to show content length when writing the access log. --- web/util/writer_proxy.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web/util/writer_proxy.go b/web/util/writer_proxy.go index 3ba08de..4406a3c 100644 --- a/web/util/writer_proxy.go +++ b/web/util/writer_proxy.go @@ -14,6 +14,8 @@ type WriterProxy interface { // Status returns the HTTP status of the request, or 0 if one has not // yet been sent. Status() int + // BytesWritten returns the total number of bytes sent to the client. + BytesWritten() int // Tee causes the response body to be written to the given io.Writer in // addition to proxying the writes through. Only one io.Writer can be // tee'd to at once: setting a second one will overwrite the first. @@ -46,6 +48,7 @@ type basicWriter struct { http.ResponseWriter wroteHeader bool code int + bytes int tee io.Writer } @@ -66,6 +69,7 @@ func (b *basicWriter) Write(buf []byte) (int, error) { err = err2 } } + b.bytes += n return n, err } func (b *basicWriter) maybeWriteHeader() { @@ -76,6 +80,9 @@ func (b *basicWriter) maybeWriteHeader() { func (b *basicWriter) Status() int { return b.code } +func (b *basicWriter) BytesWritten() int { + return b.bytes +} func (b *basicWriter) Tee(w io.Writer) { b.tee = w } From 57e752c3bcac3538485b6a74f854134ded80998a Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 2 Sep 2014 01:29:28 -0700 Subject: [PATCH 134/217] Refactor package graceful This is meant to accomplish a few things: 1. graceful no longer spawns an additional goroutine per connection. Instead, it maintains a sharded set of idle connections that a single reaper goroutine can go through when necessary. 2. graceful's connection struct has a more orthogonal set of connection state flags, replacing the harder-to-understand state machine. The underlying mechanics are largely the same, however. 3. graceful now uses the Go 1.3 ConnState API to avoid the "200-year SetReadDeadline hack." It still falls back on SetReadDeadline on Go 1.2 or where ConnState does not apply. --- graceful/conn_set.go | 80 ++++++++++++++ graceful/middleware.go | 2 +- graceful/net.go | 243 ++++++++++++++++++----------------------- graceful/net_test.go | 198 --------------------------------- graceful/serve.go | 1 + graceful/serve13.go | 41 +++++-- 6 files changed, 225 insertions(+), 340 deletions(-) create mode 100644 graceful/conn_set.go delete mode 100644 graceful/net_test.go diff --git a/graceful/conn_set.go b/graceful/conn_set.go new file mode 100644 index 0000000..c08099c --- /dev/null +++ b/graceful/conn_set.go @@ -0,0 +1,80 @@ +package graceful + +import ( + "runtime" + "sync" +) + +type connShard struct { + mu sync.Mutex + // We sort of abuse this field to also act as a "please shut down" flag. + // If it's nil, you should die at your earliest opportunity. + set map[*conn]struct{} +} + +type connSet struct { + // This is an incrementing connection counter so we round-robin + // connections across shards. Use atomic when touching it. + id uint64 + shards []*connShard +} + +var idleSet connSet + +// We pretty aggressively preallocate set entries in the hopes that we never +// have to allocate memory with the lock held. This is definitely a premature +// optimization and is probably misguided, but luckily it costs us essentially +// nothing. +const prealloc = 2048 + +func init() { + // To keep the expected contention rate constant we'd have to grow this + // as numcpu**2. In practice, CPU counts don't generally grow without + // bound, and contention is probably going to be small enough that + // nobody cares anyways. + idleSet.shards = make([]*connShard, 2*runtime.NumCPU()) + for i := range idleSet.shards { + idleSet.shards[i] = &connShard{ + set: make(map[*conn]struct{}, prealloc), + } + } +} + +func (cs connSet) markIdle(c *conn) { + c.busy = false + shard := cs.shards[int(c.id%uint64(len(cs.shards)))] + shard.mu.Lock() + if shard.set == nil { + shard.mu.Unlock() + c.die = true + } else { + shard.set[c] = struct{}{} + shard.mu.Unlock() + } +} + +func (cs connSet) markBusy(c *conn) { + c.busy = true + shard := cs.shards[int(c.id%uint64(len(cs.shards)))] + shard.mu.Lock() + if shard.set == nil { + shard.mu.Unlock() + c.die = true + } else { + delete(shard.set, c) + shard.mu.Unlock() + } +} + +func (cs connSet) killall() { + for _, shard := range cs.shards { + shard.mu.Lock() + set := shard.set + shard.set = nil + shard.mu.Unlock() + + for conn := range set { + conn.closeIfIdle() + } + } +} diff --git a/graceful/middleware.go b/graceful/middleware.go index f169891..4aace7d 100644 --- a/graceful/middleware.go +++ b/graceful/middleware.go @@ -103,7 +103,7 @@ func (f *fancyWriter) Hijack() (c net.Conn, b *bufio.ReadWriter, e error) { hj := f.basicWriter.ResponseWriter.(http.Hijacker) c, b, e = hj.Hijack() - if conn, ok := c.(hijackConn); ok { + if conn, ok := c.(*conn); ok { c = conn.hijack() } diff --git a/graceful/net.go b/graceful/net.go index b3f124a..e667d05 100644 --- a/graceful/net.go +++ b/graceful/net.go @@ -4,6 +4,7 @@ import ( "io" "net" "sync" + "sync/atomic" "time" ) @@ -11,10 +12,6 @@ type listener struct { net.Listener } -type gracefulConn interface { - gracefulShutdown() -} - // WrapListener wraps an arbitrary net.Listener for use with graceful shutdowns. // All net.Conn's Accept()ed by this listener will be auto-wrapped as if // WrapConn() were called on them. @@ -24,11 +21,17 @@ func WrapListener(l net.Listener) net.Listener { func (l listener) Accept() (net.Conn, error) { conn, err := l.Listener.Accept() - if err != nil { - return nil, err - } + return WrapConn(conn), err +} - return WrapConn(conn), nil +type conn struct { + mu sync.Mutex + cs *connSet + net.Conn + id uint64 + busy, die bool + dead bool + hijacked bool } /* @@ -39,10 +42,9 @@ terminating the process. In order to use this function, you must call SetReadDeadline() before the call to Read() you might make to read a new request off the wire. The connection is eligible for abrupt closing at any point between when the call to -SetReadDeadline() returns and when the call to Read returns with new data. It -does not matter what deadline is given to SetReadDeadline()--the default HTTP -server provided by this package sets a deadline far into the future when a -deadline is not provided, for instance. +SetReadDeadline() returns and when the call to Read returns with new data. It +does not matter what deadline is given to SetReadDeadline()--if a deadline is +inappropriate, providing one extremely far into the future will suffice. Unfortunately, this means that it's difficult to use SetReadDeadline() in a great many perfectly reasonable circumstances, such as to extend a deadline @@ -50,152 +52,125 @@ after more data has been read, without the connection being eligible for "graceful" termination at an undesirable time. Since this package was written explicitly to target net/http, which does not as of this writing do any of this, fixing the semantics here does not seem especially urgent. - -As an optimization for net/http over TCP, if the input connection supports the -ReadFrom() function, the returned connection will as well. This allows the net -package to use sendfile(2) on certain platforms in certain circumstances. */ func WrapConn(c net.Conn) net.Conn { - wg.Add(1) - - nc := conn{ - Conn: c, - closing: make(chan struct{}), + if c == nil { + return nil } - if _, ok := c.(io.ReaderFrom); ok { - c = &sendfile{nc} - } else { - c = &nc + wg.Add(1) + return &conn{ + Conn: c, + id: atomic.AddUint64(&idleSet.id, 1), } +} - go c.(gracefulConn).gracefulShutdown() +func (c *conn) Read(b []byte) (n int, err error) { + c.mu.Lock() + if !c.hijacked { + defer func() { + c.mu.Lock() + if c.hijacked { + // It's a little unclear to me how this case + // would happen, but we *did* drop the lock, so + // let's play it safe. + return + } + + if c.dead { + // Dead sockets don't tell tales. This is to + // prevent the case where a Read manages to suck + // an entire request off the wire in a race with + // someone trying to close idle connections. + // Whoever grabs the conn lock first wins, and + // if that's the closing process, we need to + // "take back" the read. + n = 0 + err = io.EOF + } else { + idleSet.markBusy(c) + } + c.mu.Unlock() + }() + } + c.mu.Unlock() - return c + return c.Conn.Read(b) } -type connstate int - -/* -State diagram. (Waiting) is the starting state. - -(Waiting) -----Read()-----> Working ---+ - | ^ / | ^ Read() - | \ / | +----+ - kill SetReadDeadline() kill - | | +-----+ - V V V Read() - Dead <-SetReadDeadline()-- Dying ----+ - ^ - | - +--Close()--- [from any state] +func (c *conn) SetReadDeadline(t time.Time) error { + c.mu.Lock() + if !c.hijacked { + defer c.markIdle() + } + c.mu.Unlock() + return c.Conn.SetReadDeadline(t) +} -*/ +func (c *conn) Close() error { + kill := false + c.mu.Lock() + kill, c.dead = !c.dead, true + idleSet.markBusy(c) + c.mu.Unlock() + + if kill { + defer wg.Done() + } + return c.Conn.Close() +} -const ( - // Waiting for more data, and eligible for killing - csWaiting connstate = iota - // In the middle of a connection - csWorking - // Kill has been requested, but waiting on request to finish up - csDying - // Connection is gone forever. Also used when a connection gets hijacked - csDead -) +type writerOnly struct { + w io.Writer +} -type conn struct { - net.Conn - m sync.Mutex - state connstate - closing chan struct{} +func (w writerOnly) Write(buf []byte) (int, error) { + return w.w.Write(buf) } -type sendfile struct{ conn } -func (c *conn) gracefulShutdown() { - select { - case <-kill: - case <-c.closing: - return - } - c.m.Lock() - defer c.m.Unlock() - - switch c.state { - case csWaiting: - c.unlockedClose(true) - case csWorking: - c.state = csDying +func (c *conn) ReadFrom(r io.Reader) (int64, error) { + if rf, ok := c.Conn.(io.ReaderFrom); ok { + return rf.ReadFrom(r) } + return io.Copy(writerOnly{c}, r) } -func (c *conn) unlockedClose(closeConn bool) { - if closeConn { +func (c *conn) markIdle() { + kill := false + c.mu.Lock() + idleSet.markIdle(c) + if c.die { + kill, c.dead = !c.dead, true + } + c.mu.Unlock() + + if kill { + defer wg.Done() c.Conn.Close() } - close(c.closing) - wg.Done() - c.state = csDead -} -// We do some hijinks to support hijacking. The semantics here is that any -// connection that gets hijacked is dead to us: we return the raw net.Conn and -// stop tracking the connection entirely. -type hijackConn interface { - hijack() net.Conn } -func (c *conn) hijack() net.Conn { - c.m.Lock() - defer c.m.Unlock() - if c.state != csDead { - close(c.closing) - wg.Done() - c.state = csDead +func (c *conn) closeIfIdle() { + kill := false + c.mu.Lock() + c.die = true + if !c.busy && !c.hijacked { + kill, c.dead = !c.dead, true } - return c.Conn -} - -func (c *conn) Read(b []byte) (n int, err error) { - defer func() { - c.m.Lock() - defer c.m.Unlock() - - if c.state == csWaiting { - c.state = csWorking - } else if c.state == csDead { - n = 0 - err = io.EOF - } - }() + c.mu.Unlock() - return c.Conn.Read(b) -} -func (c *conn) Close() error { - defer func() { - c.m.Lock() - defer c.m.Unlock() - - if c.state != csDead { - c.unlockedClose(false) - } - }() - return c.Conn.Close() -} -func (c *conn) SetReadDeadline(t time.Time) error { - defer func() { - c.m.Lock() - defer c.m.Unlock() - switch c.state { - case csDying: - c.unlockedClose(false) - case csWorking: - c.state = csWaiting - } - }() - return c.Conn.SetReadDeadline(t) + if kill { + defer wg.Done() + c.Conn.Close() + } } -func (s *sendfile) ReadFrom(r io.Reader) (int64, error) { - // conn.Conn.KHAAAAAAAANNNNNN - return s.conn.Conn.(io.ReaderFrom).ReadFrom(r) +func (c *conn) hijack() net.Conn { + c.mu.Lock() + idleSet.markBusy(c) + c.hijacked = true + c.mu.Unlock() + + return c.Conn } diff --git a/graceful/net_test.go b/graceful/net_test.go deleted file mode 100644 index d6e7208..0000000 --- a/graceful/net_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package graceful - -import ( - "io" - "net" - "strings" - "testing" - "time" -) - -var b = make([]byte, 0) - -func connify(c net.Conn) *conn { - switch c.(type) { - case (*conn): - return c.(*conn) - case (*sendfile): - return &c.(*sendfile).conn - default: - panic("IDK") - } -} - -func assertState(t *testing.T, n net.Conn, st connstate) { - c := connify(n) - c.m.Lock() - defer c.m.Unlock() - if c.state != st { - t.Fatalf("conn was %v, but expected %v", c.state, st) - } -} - -// Not super happy about making the tests dependent on the passing of time, but -// I'm not really sure what else to do. - -func expectCall(t *testing.T, ch <-chan struct{}, name string) { - select { - case <-ch: - case <-time.After(5 * time.Millisecond): - t.Fatalf("Expected call to %s", name) - } -} - -func TestCounting(t *testing.T) { - kill = make(chan struct{}) - c := WrapConn(fakeConn{}) - ch := make(chan struct{}) - - go func() { - wg.Wait() - ch <- struct{}{} - }() - - select { - case <-ch: - t.Fatal("Expected connection to keep us from quitting") - case <-time.After(5 * time.Millisecond): - } - - c.Close() - expectCall(t, ch, "wg.Wait()") -} - -func TestStateTransitions1(t *testing.T) { - kill = make(chan struct{}) - ch := make(chan struct{}) - - onclose := make(chan struct{}) - read := make(chan struct{}) - deadline := make(chan struct{}) - c := WrapConn(fakeConn{ - onClose: func() { - onclose <- struct{}{} - }, - onRead: func() { - read <- struct{}{} - }, - onSetReadDeadline: func() { - deadline <- struct{}{} - }, - }) - - go func() { - wg.Wait() - ch <- struct{}{} - }() - - assertState(t, c, csWaiting) - - // Waiting + Read() = Working - go c.Read(b) - expectCall(t, read, "c.Read()") - assertState(t, c, csWorking) - - // Working + SetReadDeadline() = Waiting - go c.SetReadDeadline(time.Now()) - expectCall(t, deadline, "c.SetReadDeadline()") - assertState(t, c, csWaiting) - - // Waiting + kill = Dead - close(kill) - expectCall(t, onclose, "c.Close()") - assertState(t, c, csDead) - - expectCall(t, ch, "wg.Wait()") -} - -func TestStateTransitions2(t *testing.T) { - kill = make(chan struct{}) - ch := make(chan struct{}) - onclose := make(chan struct{}) - read := make(chan struct{}) - deadline := make(chan struct{}) - c := WrapConn(fakeConn{ - onClose: func() { - onclose <- struct{}{} - }, - onRead: func() { - read <- struct{}{} - }, - onSetReadDeadline: func() { - deadline <- struct{}{} - }, - }) - - go func() { - wg.Wait() - ch <- struct{}{} - }() - - assertState(t, c, csWaiting) - - // Waiting + Read() = Working - go c.Read(b) - expectCall(t, read, "c.Read()") - assertState(t, c, csWorking) - - // Working + Read() = Working - go c.Read(b) - expectCall(t, read, "c.Read()") - assertState(t, c, csWorking) - - // Working + kill = Dying - close(kill) - time.Sleep(5 * time.Millisecond) - assertState(t, c, csDying) - - // Dying + Read() = Dying - go c.Read(b) - expectCall(t, read, "c.Read()") - assertState(t, c, csDying) - - // Dying + SetReadDeadline() = Dead - go c.SetReadDeadline(time.Now()) - expectCall(t, deadline, "c.SetReadDeadline()") - assertState(t, c, csDead) - - expectCall(t, ch, "wg.Wait()") -} - -func TestHijack(t *testing.T) { - kill = make(chan struct{}) - fake := fakeConn{} - c := WrapConn(fake) - ch := make(chan struct{}) - - go func() { - wg.Wait() - ch <- struct{}{} - }() - - cc := connify(c) - if _, ok := cc.hijack().(fakeConn); !ok { - t.Error("Expected original connection back out") - } - assertState(t, c, csDead) - expectCall(t, ch, "wg.Wait()") -} - -type fakeSendfile struct { - fakeConn -} - -func (f fakeSendfile) ReadFrom(r io.Reader) (int64, error) { - return 0, nil -} - -func TestReadFrom(t *testing.T) { - kill = make(chan struct{}) - c := WrapConn(fakeSendfile{}) - r := strings.NewReader("Hello world") - - if rf, ok := c.(io.ReaderFrom); ok { - rf.ReadFrom(r) - } else { - t.Fatal("Expected a ReaderFrom in return") - } -} diff --git a/graceful/serve.go b/graceful/serve.go index 8746075..626f807 100644 --- a/graceful/serve.go +++ b/graceful/serve.go @@ -15,6 +15,7 @@ func (srv *Server) Serve(l net.Listener) error { go func() { <-kill l.Close() + idleSet.killall() }() l = WrapListener(l) diff --git a/graceful/serve13.go b/graceful/serve13.go index 41e9f52..e0acd18 100644 --- a/graceful/serve13.go +++ b/graceful/serve13.go @@ -5,12 +5,8 @@ package graceful import ( "net" "net/http" - "time" ) -// About 200 years, also known as "forever" -const forever time.Duration = 200 * 365 * 24 * time.Hour - func (srv *Server) Serve(l net.Listener) error { l = WrapListener(l) @@ -19,14 +15,45 @@ func (srv *Server) Serve(l net.Listener) error { // and it's nice to keep our sketching to ourselves. shadow := *(*http.Server)(srv) - if shadow.ReadTimeout == 0 { - shadow.ReadTimeout = forever + cs := shadow.ConnState + shadow.ConnState = func(nc net.Conn, s http.ConnState) { + if c, ok := nc.(*conn); ok { + // There are a few other states defined, most notably + // StateActive. Unfortunately it doesn't look like it's + // possible to make use of StateActive to implement + // graceful shutdown, since StateActive is set after a + // complete request has been read off the wire with an + // intent to process it. If we were to race a graceful + // shutdown against a connection that was just read off + // the wire (but not yet in StateActive), we would + // accidentally close the connection out from underneath + // an active request. + // + // We already needed to work around this for Go 1.2 by + // shimming out a full net.Conn object, so we can just + // fall back to the old behavior there. + // + // I started a golang-nuts thread about this here: + // https://groups.google.com/forum/#!topic/golang-nuts/Xi8yjBGWfCQ + // I'd be very eager to find a better way to do this, so + // reach out to me if you have any ideas. + switch s { + case http.StateIdle: + c.markIdle() + case http.StateHijacked: + c.hijack() + } + } + if cs != nil { + cs(nc, s) + } } go func() { <-kill - shadow.SetKeepAlivesEnabled(false) l.Close() + shadow.SetKeepAlivesEnabled(false) + idleSet.killall() }() err := shadow.Serve(l) From 119938d4b94fdf713efb3340ebd4ba5f3286f520 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Wed, 8 Oct 2014 00:58:55 -0700 Subject: [PATCH 135/217] Add bind.Sniff to expose default selection logic This allows users who do not wish to use bind.WithFlag() to still take advantage of bind's magical listener selection logic. Fixes #68. --- bind/bind.go | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/bind/bind.go b/bind/bind.go index 9228f5a..8e783bf 100644 --- a/bind/bind.go +++ b/bind/bind.go @@ -52,14 +52,8 @@ func init() { // of the default socket will be reasonable for use in your circumstance. func WithFlag() { defaultBind := ":8000" - if bind := os.Getenv("GOJI_BIND"); bind != "" { - defaultBind = bind - } else if usingEinhorn() { - defaultBind = "einhorn@0" - } else if usingSystemd() { - defaultBind = "fd@3" - } else if port := os.Getenv("PORT"); port != "" { - defaultBind = ":" + port + if s := Sniff(); s != "" { + defaultBind = s } flag.StringVar(&bind, "bind", defaultBind, `Address to bind on. If this value has a colon, as in ":8000" or @@ -74,6 +68,24 @@ func WithFlag() { (systemd), and ":8000" (fallback) based on its environment.`) } +// Sniff attempts to select a sensible default bind string by examining its +// environment. It examines the GOJI_BIND environment variable, Einhorn, +// systemd, and the PORT environment variable, in that order, selecting the +// first plausible option. It returns the empty string if no sensible default +// could be extracted from the environment. +func Sniff() string { + if bind := os.Getenv("GOJI_BIND"); bind != "" { + return bind + } else if usingEinhorn() { + return "einhorn@0" + } else if usingSystemd() { + return "fd@3" + } else if port := os.Getenv("PORT"); port != "" { + return ":" + port + } + return "" +} + func listenTo(bind string) (net.Listener, error) { if strings.Contains(bind, ":") { return net.Listen("tcp", bind) From 04b25f26e54f4b42c26d746eb83ab8b41e7dd272 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 26 Oct 2014 15:20:29 -0700 Subject: [PATCH 136/217] Don't expose the internal routeMachine --- web/router.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/web/router.go b/web/router.go index cf93d93..952bf4d 100644 --- a/web/router.go +++ b/web/router.go @@ -230,7 +230,11 @@ func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (meth // after all the routes have been added, and will be called automatically for // you (at some performance cost on the first request) if you do not call it // explicitly. -func (rt *router) Compile() *routeMachine { +func (rt *router) Compile() { + rt.compile() +} + +func (rt *router) compile() *routeMachine { rt.lock.Lock() defer rt.lock.Unlock() sm := routeMachine{ @@ -244,7 +248,7 @@ func (rt *router) Compile() *routeMachine { func (rt *router) route(c *C, w http.ResponseWriter, r *http.Request) { rm := rt.getMachine() if rm == nil { - rm = rt.Compile() + rm = rt.compile() } methods, ok := rm.route(c, w, r) From 2c106c958bc0b7ac2488692330c77af8711c9c69 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 26 Oct 2014 16:13:27 -0700 Subject: [PATCH 137/217] Start moving away from struct embedding Instead of using struct embedding to build web.Mux, start moving towards explicit mappings. This doesn't actually change the public API of web.Mux, but feels a little cleaner to me. The longer-term thing here is to get rid of the functions defined on Muxes in the public documentation that are defined on "rt *Mux", which is just plain ugly. --- web/middleware.go | 43 ++++++++++--------------------------------- web/mux.go | 47 ++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 50 insertions(+), 40 deletions(-) diff --git a/web/middleware.go b/web/middleware.go index 6ebf379..db766ad 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -7,11 +7,16 @@ import ( "sync" ) +// mLayer is a single middleware stack layer. It contains a canonicalized +// middleware representation, as well as the original function as passed to us. type mLayer struct { fn func(*C, http.Handler) http.Handler orig interface{} } +// mStack is an entire middleware stack. It contains a slice of middleware +// layers (outermost first) protected by a mutex, a cache of pre-built stack +// instances, and a final routing function. type mStack struct { lock sync.Mutex stack []mLayer @@ -24,21 +29,11 @@ type internalRouter interface { } /* -Constructing a middleware stack involves a lot of allocations: at the very least -each layer will have to close over the layer after (inside) it, and perhaps a -context object. Instead of doing this on every request, let's cache fully -assembled middleware stacks (the "c" stands for "cached"). - -A lot of the complexity here (in particular the "pool" parameter, and the -behavior of release() and invalidate() below) is due to the fact that when the -middleware stack is mutated we need to create a "cache barrier," where no -cStack created before the middleware stack mutation is returned to the active -cache pool (and is therefore eligible for subsequent reuse). The way we do this -is a bit ugly: each cStack maintains a pointer to the pool it originally came -from, and will only return itself to that pool. If the mStack's pool has been -rotated since then (meaning that this cStack is invalid), it will either try -(and likely fail) to insert itself into the stale pool, or it will drop the -cStack on the floor. +cStack is a cached middleware stack instance. Constructing a middleware stack +involves a lot of allocations: at the very least each layer will have to close +over the layer after (inside) it and a stack N levels deep will incur at least N +separate allocations. Instead of doing this on every request, we keep a pool of +pre-built stacks around for reuse. */ type cStack struct { C @@ -120,11 +115,6 @@ func (m *mStack) release(cs *cStack) { cs.pool = nil } -// Append the given middleware to the middleware stack. See the documentation -// for type Mux for a list of valid middleware types. -// -// No attempt is made to enforce the uniqueness of middlewares. It is illegal to -// call this function concurrently with active requests. func (m *mStack) Use(middleware interface{}) { m.lock.Lock() defer m.lock.Unlock() @@ -132,13 +122,6 @@ func (m *mStack) Use(middleware interface{}) { m.invalidate() } -// Insert the given middleware immediately before a given existing middleware in -// the stack. See the documentation for type Mux for a list of valid middleware -// types. Returns an error if no middleware has the name given by "before." -// -// No attempt is made to enforce the uniqueness of middlewares. If the insertion -// point is ambiguous, the first (outermost) one is chosen. It is illegal to -// call this function concurrently with active requests. func (m *mStack) Insert(middleware, before interface{}) error { m.lock.Lock() defer m.lock.Unlock() @@ -156,12 +139,6 @@ func (m *mStack) Insert(middleware, before interface{}) error { return nil } -// Remove the given middleware from the middleware stack. Returns an error if -// no such middleware can be found. -// -// If the name of the middleware to delete is ambiguous, the first (outermost) -// one is chosen. It is illegal to call this function concurrently with active -// requests. func (m *mStack) Abandon(middleware interface{}) error { m.lock.Lock() defer m.lock.Unlock() diff --git a/web/mux.go b/web/mux.go index de5a032..8c38551 100644 --- a/web/mux.go +++ b/web/mux.go @@ -52,14 +52,14 @@ Handler must be one of the following types: - func(c web.C, w http.ResponseWriter, r *http.Request) */ type Mux struct { - mStack + ms mStack router } // New creates a new Mux without any routes or middleware. func New() *Mux { mux := Mux{ - mStack: mStack{ + ms: mStack{ stack: make([]mLayer, 0), pool: makeCPool(), }, @@ -68,20 +68,53 @@ func New() *Mux { notFound: parseHandler(http.NotFound), }, } - mux.mStack.router = &mux.router + mux.ms.router = &mux.router return &mux } +// ServeHTTP processes HTTP requests. It make Muxes satisfy net/http.Handler. func (m *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) { - stack := m.mStack.alloc() + stack := m.ms.alloc() stack.ServeHTTP(w, r) - m.mStack.release(stack) + m.ms.release(stack) } // ServeHTTPC creates a context dependent request with the given Mux. Satisfies // the web.Handler interface. func (m *Mux) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { - stack := m.mStack.alloc() + stack := m.ms.alloc() stack.ServeHTTPC(c, w, r) - m.mStack.release(stack) + m.ms.release(stack) +} + +// Middleware Stack functions + +// Append the given middleware to the middleware stack. See the documentation +// for type Mux for a list of valid middleware types. +// +// No attempt is made to enforce the uniqueness of middlewares. It is illegal to +// call this function concurrently with active requests. +func (m *Mux) Use(middleware interface{}) { + m.ms.Use(middleware) +} + +// Insert the given middleware immediately before a given existing middleware in +// the stack. See the documentation for type Mux for a list of valid middleware +// types. Returns an error if no middleware has the name given by "before." +// +// No attempt is made to enforce the uniqueness of middlewares. If the insertion +// point is ambiguous, the first (outermost) one is chosen. It is illegal to +// call this function concurrently with active requests. +func (m *Mux) Insert(middleware, before interface{}) error { + return m.ms.Insert(middleware, before) +} + +// Remove the given middleware from the middleware stack. Returns an error if +// no such middleware can be found. +// +// If the name of the middleware to delete is ambiguous, the first (outermost) +// one is chosen. It is illegal to call this function concurrently with active +// requests. +func (m *Mux) Abandon(middleware interface{}) error { + return m.ms.Abandon(middleware) } From f2926972236e62bb85a77c9ed50514d23fd5b294 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 26 Oct 2014 16:25:25 -0700 Subject: [PATCH 138/217] Put more methods on Mux This fixes the documentation wart where half of the methods were defined on "rt *Mux". It doesn't change the public API. --- web/mux.go | 111 +++++++++++++++++++++++++++++++++++++++++++-- web/router.go | 107 ------------------------------------------- web/router_test.go | 101 +++++++++++++++++++---------------------- 3 files changed, 155 insertions(+), 164 deletions(-) diff --git a/web/mux.go b/web/mux.go index 8c38551..7fafa8e 100644 --- a/web/mux.go +++ b/web/mux.go @@ -53,7 +53,7 @@ Handler must be one of the following types: */ type Mux struct { ms mStack - router + rt router } // New creates a new Mux without any routes or middleware. @@ -63,12 +63,12 @@ func New() *Mux { stack: make([]mLayer, 0), pool: makeCPool(), }, - router: router{ + rt: router{ routes: make([]route, 0), notFound: parseHandler(http.NotFound), }, } - mux.ms.router = &mux.router + mux.ms.router = &mux.rt return &mux } @@ -118,3 +118,108 @@ func (m *Mux) Insert(middleware, before interface{}) error { func (m *Mux) Abandon(middleware interface{}) error { return m.ms.Abandon(middleware) } + +// Router functions + +/* +Dispatch to the given handler when the pattern matches, regardless of HTTP +method. See the documentation for type Mux for a description of what types are +accepted for pattern and handler. + +This method is commonly used to implement sub-routing: an admin application, for +instance, can expose a single handler that is attached to the main Mux by +calling Handle("/admin*", adminHandler) or similar. Note that this function +doesn't strip this prefix from the path before forwarding it on (e.g., the +handler will see the full path, including the "/admin" part), but this +functionality can easily be performed by an extra middleware layer. +*/ +func (m *Mux) Handle(pattern interface{}, handler interface{}) { + m.rt.handleUntyped(pattern, mALL, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// CONNECT. See the documentation for type Mux for a description of what types +// are accepted for pattern and handler. +func (m *Mux) Connect(pattern interface{}, handler interface{}) { + m.rt.handleUntyped(pattern, mCONNECT, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// DELETE. See the documentation for type Mux for a description of what types +// are accepted for pattern and handler. +func (m *Mux) Delete(pattern interface{}, handler interface{}) { + m.rt.handleUntyped(pattern, mDELETE, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// GET. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +// +// All GET handlers also transparently serve HEAD requests, since net/http will +// take care of all the fiddly bits for you. If you wish to provide an alternate +// implementation of HEAD, you should add a handler explicitly and place it +// above your GET handler. +func (m *Mux) Get(pattern interface{}, handler interface{}) { + m.rt.handleUntyped(pattern, mGET|mHEAD, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// HEAD. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +func (m *Mux) Head(pattern interface{}, handler interface{}) { + m.rt.handleUntyped(pattern, mHEAD, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// OPTIONS. See the documentation for type Mux for a description of what types +// are accepted for pattern and handler. +func (m *Mux) Options(pattern interface{}, handler interface{}) { + m.rt.handleUntyped(pattern, mOPTIONS, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// PATCH. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +func (m *Mux) Patch(pattern interface{}, handler interface{}) { + m.rt.handleUntyped(pattern, mPATCH, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// POST. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +func (m *Mux) Post(pattern interface{}, handler interface{}) { + m.rt.handleUntyped(pattern, mPOST, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// PUT. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +func (m *Mux) Put(pattern interface{}, handler interface{}) { + m.rt.handleUntyped(pattern, mPUT, handler) +} + +// Dispatch to the given handler when the pattern matches and the HTTP method is +// TRACE. See the documentation for type Mux for a description of what types are +// accepted for pattern and handler. +func (m *Mux) Trace(pattern interface{}, handler interface{}) { + m.rt.handleUntyped(pattern, mTRACE, handler) +} + +// Set the fallback (i.e., 404) handler for this mux. See the documentation for +// type Mux for a description of what types are accepted for handler. +// +// As a convenience, the context environment variable "goji.web.validMethods" +// (also available as the constant ValidMethodsKey) will be set to the list of +// HTTP methods that could have been routed had they been provided on an +// otherwise identical request. +func (m *Mux) NotFound(handler interface{}) { + m.rt.notFound = parseHandler(handler) +} + +// Compile the list of routes into bytecode. This only needs to be done once +// after all the routes have been added, and will be called automatically for +// you (at some performance cost on the first request) if you do not call it +// explicitly. +func (rt *router) Compile() { + rt.compile() +} diff --git a/web/router.go b/web/router.go index 952bf4d..f96f034 100644 --- a/web/router.go +++ b/web/router.go @@ -226,14 +226,6 @@ func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (meth return methods, false } -// Compile the list of routes into bytecode. This only needs to be done once -// after all the routes have been added, and will be called automatically for -// you (at some performance cost on the first request) if you do not call it -// explicitly. -func (rt *router) Compile() { - rt.compile() -} - func (rt *router) compile() *routeMachine { rt.lock.Lock() defer rt.lock.Unlock() @@ -313,102 +305,3 @@ func (rt *router) handle(p Pattern, m method, h Handler) { rt.setMachine(nil) rt.routes = newRoutes } - -// This is a bit silly, but I've renamed the method receivers in the public -// functions here "m" instead of the standard "rt", since they will eventually -// be shown on the documentation for the Mux that they are included in. - -/* -Dispatch to the given handler when the pattern matches, regardless of HTTP -method. See the documentation for type Mux for a description of what types are -accepted for pattern and handler. - -This method is commonly used to implement sub-routing: an admin application, for -instance, can expose a single handler that is attached to the main Mux by -calling Handle("/admin*", adminHandler) or similar. Note that this function -doesn't strip this prefix from the path before forwarding it on (e.g., the -handler will see the full path, including the "/admin" part), but this -functionality can easily be performed by an extra middleware layer. -*/ -func (rt *router) Handle(pattern interface{}, handler interface{}) { - rt.handleUntyped(pattern, mALL, handler) -} - -// Dispatch to the given handler when the pattern matches and the HTTP method is -// CONNECT. See the documentation for type Mux for a description of what types -// are accepted for pattern and handler. -func (rt *router) Connect(pattern interface{}, handler interface{}) { - rt.handleUntyped(pattern, mCONNECT, handler) -} - -// Dispatch to the given handler when the pattern matches and the HTTP method is -// DELETE. See the documentation for type Mux for a description of what types -// are accepted for pattern and handler. -func (rt *router) Delete(pattern interface{}, handler interface{}) { - rt.handleUntyped(pattern, mDELETE, handler) -} - -// Dispatch to the given handler when the pattern matches and the HTTP method is -// GET. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. -// -// All GET handlers also transparently serve HEAD requests, since net/http will -// take care of all the fiddly bits for you. If you wish to provide an alternate -// implementation of HEAD, you should add a handler explicitly and place it -// above your GET handler. -func (rt *router) Get(pattern interface{}, handler interface{}) { - rt.handleUntyped(pattern, mGET|mHEAD, handler) -} - -// Dispatch to the given handler when the pattern matches and the HTTP method is -// HEAD. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. -func (rt *router) Head(pattern interface{}, handler interface{}) { - rt.handleUntyped(pattern, mHEAD, handler) -} - -// Dispatch to the given handler when the pattern matches and the HTTP method is -// OPTIONS. See the documentation for type Mux for a description of what types -// are accepted for pattern and handler. -func (rt *router) Options(pattern interface{}, handler interface{}) { - rt.handleUntyped(pattern, mOPTIONS, handler) -} - -// Dispatch to the given handler when the pattern matches and the HTTP method is -// PATCH. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. -func (rt *router) Patch(pattern interface{}, handler interface{}) { - rt.handleUntyped(pattern, mPATCH, handler) -} - -// Dispatch to the given handler when the pattern matches and the HTTP method is -// POST. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. -func (rt *router) Post(pattern interface{}, handler interface{}) { - rt.handleUntyped(pattern, mPOST, handler) -} - -// Dispatch to the given handler when the pattern matches and the HTTP method is -// PUT. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. -func (rt *router) Put(pattern interface{}, handler interface{}) { - rt.handleUntyped(pattern, mPUT, handler) -} - -// Dispatch to the given handler when the pattern matches and the HTTP method is -// TRACE. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. -func (rt *router) Trace(pattern interface{}, handler interface{}) { - rt.handleUntyped(pattern, mTRACE, handler) -} - -// Set the fallback (i.e., 404) handler for this mux. See the documentation for -// type Mux for a description of what types are accepted for handler. -// -// As a convenience, the context environment variable "goji.web.validMethods" -// (also available as the constant ValidMethodsKey) will be set to the list of -// HTTP methods that could have been routed had they been provided on an -// otherwise identical request. -func (rt *router) NotFound(handler interface{}) { - rt.notFound = parseHandler(handler) -} diff --git a/web/router_test.go b/web/router_test.go index 59955cb..b4c9730 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -11,13 +11,6 @@ import ( // These tests can probably be DRY'd up a bunch -func makeRouter() *router { - return &router{ - routes: make([]route, 0), - notFound: parseHandler(http.NotFound), - } -} - func chHandler(ch chan string, s string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ch <- s @@ -29,24 +22,24 @@ var methods = []string{"CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH", func TestMethods(t *testing.T) { t.Parallel() - rt := makeRouter() + m := New() ch := make(chan string, 1) - rt.Connect("/", chHandler(ch, "CONNECT")) - rt.Delete("/", chHandler(ch, "DELETE")) - rt.Head("/", chHandler(ch, "HEAD")) - rt.Get("/", chHandler(ch, "GET")) - rt.Options("/", chHandler(ch, "OPTIONS")) - rt.Patch("/", chHandler(ch, "PATCH")) - rt.Post("/", chHandler(ch, "POST")) - rt.Put("/", chHandler(ch, "PUT")) - rt.Trace("/", chHandler(ch, "TRACE")) - rt.Handle("/", chHandler(ch, "OTHER")) + m.Connect("/", chHandler(ch, "CONNECT")) + m.Delete("/", chHandler(ch, "DELETE")) + m.Head("/", chHandler(ch, "HEAD")) + m.Get("/", chHandler(ch, "GET")) + m.Options("/", chHandler(ch, "OPTIONS")) + m.Patch("/", chHandler(ch, "PATCH")) + m.Post("/", chHandler(ch, "POST")) + m.Put("/", chHandler(ch, "PUT")) + m.Trace("/", chHandler(ch, "TRACE")) + m.Handle("/", chHandler(ch, "OTHER")) for _, method := range methods { r, _ := http.NewRequest(method, "/", nil) w := httptest.NewRecorder() - rt.route(&C{}, w, r) + m.ServeHTTP(w, r) select { case val := <-ch: if val != method { @@ -74,12 +67,12 @@ var _ Pattern = testPattern{} func TestPatternTypes(t *testing.T) { t.Parallel() - rt := makeRouter() + m := New() - rt.Get("/hello/carl", http.NotFound) - rt.Get("/hello/:name", http.NotFound) - rt.Get(regexp.MustCompile(`^/hello/(?P.+)$`), http.NotFound) - rt.Get(testPattern{}, http.NotFound) + m.Get("/hello/carl", http.NotFound) + m.Get("/hello/:name", http.NotFound) + m.Get(regexp.MustCompile(`^/hello/(?P.+)$`), http.NotFound) + m.Get(testPattern{}, http.NotFound) } type testHandler chan string @@ -101,27 +94,27 @@ var testHandlerTable = map[string]string{ func TestHandlerTypes(t *testing.T) { t.Parallel() - rt := makeRouter() + m := New() ch := make(chan string, 1) - rt.Get("/a", func(w http.ResponseWriter, r *http.Request) { + m.Get("/a", func(w http.ResponseWriter, r *http.Request) { ch <- "http fn" }) - rt.Get("/b", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m.Get("/b", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ch <- "http handler" })) - rt.Get("/c", func(c C, w http.ResponseWriter, r *http.Request) { + m.Get("/c", func(c C, w http.ResponseWriter, r *http.Request) { ch <- "web fn" }) - rt.Get("/d", HandlerFunc(func(c C, w http.ResponseWriter, r *http.Request) { + m.Get("/d", HandlerFunc(func(c C, w http.ResponseWriter, r *http.Request) { ch <- "web handler" })) - rt.Get("/e", testHandler(ch)) + m.Get("/e", testHandler(ch)) for route, response := range testHandlerTable { r, _ := http.NewRequest("GET", route, nil) w := httptest.NewRecorder() - rt.route(&C{}, w, r) + m.ServeHTTP(w, r) select { case resp := <-ch: if resp != response { @@ -191,10 +184,10 @@ var _ http.Handler = rsPattern{} func TestRouteSelection(t *testing.T) { t.Parallel() - rt := makeRouter() + m := New() counter := 0 ichan := make(chan int, 1) - rt.NotFound(func(w http.ResponseWriter, r *http.Request) { + m.NotFound(func(w http.ResponseWriter, r *http.Request) { ichan <- -1 }) @@ -205,7 +198,7 @@ func TestRouteSelection(t *testing.T) { prefix: s, ichan: ichan, } - rt.Get(pat, pat) + m.Get(pat, pat) } for _, test := range rsTests { @@ -213,7 +206,7 @@ func TestRouteSelection(t *testing.T) { for counter, n = range test.results { r, _ := http.NewRequest("GET", test.key, nil) w := httptest.NewRecorder() - rt.route(&C{}, w, r) + m.ServeHTTP(w, r) actual := <-ichan if n != actual { t.Errorf("Expected %q @ %d to be %d, got %d", @@ -225,22 +218,22 @@ func TestRouteSelection(t *testing.T) { func TestNotFound(t *testing.T) { t.Parallel() - rt := makeRouter() + m := New() r, _ := http.NewRequest("post", "/", nil) w := httptest.NewRecorder() - rt.route(&C{}, w, r) + m.ServeHTTP(w, r) if w.Code != 404 { t.Errorf("Expected 404, got %d", w.Code) } - rt.NotFound(func(w http.ResponseWriter, r *http.Request) { + m.NotFound(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "I'm a teapot!", http.StatusTeapot) }) r, _ = http.NewRequest("POST", "/", nil) w = httptest.NewRecorder() - rt.route(&C{}, w, r) + m.ServeHTTP(w, r) if w.Code != http.StatusTeapot { t.Errorf("Expected a teapot, got %d", w.Code) } @@ -248,16 +241,16 @@ func TestNotFound(t *testing.T) { func TestPrefix(t *testing.T) { t.Parallel() - rt := makeRouter() + m := New() ch := make(chan string, 1) - rt.Handle("/hello*", func(w http.ResponseWriter, r *http.Request) { + m.Handle("/hello*", func(w http.ResponseWriter, r *http.Request) { ch <- r.URL.Path }) r, _ := http.NewRequest("GET", "/hello/world", nil) w := httptest.NewRecorder() - rt.route(&C{}, w, r) + m.ServeHTTP(w, r) select { case val := <-ch: if val != "/hello/world" { @@ -278,10 +271,10 @@ var validMethodsTable = map[string][]string{ func TestValidMethods(t *testing.T) { t.Parallel() - rt := makeRouter() + m := New() ch := make(chan []string, 1) - rt.NotFound(func(c C, w http.ResponseWriter, r *http.Request) { + m.NotFound(func(c C, w http.ResponseWriter, r *http.Request) { if c.Env == nil { ch <- []string{} return @@ -294,19 +287,19 @@ func TestValidMethods(t *testing.T) { ch <- methods.([]string) }) - rt.Get("/hello/carl", http.NotFound) - rt.Post("/hello/carl", http.NotFound) - rt.Head("/hello/bob", http.NotFound) - rt.Get("/hello/:name", http.NotFound) - rt.Put("/hello/:name", http.NotFound) - rt.Patch("/hello/:name", http.NotFound) - rt.Get("/:greet/carl", http.NotFound) - rt.Put("/:greet/carl", http.NotFound) - rt.Delete("/:greet/:anyone", http.NotFound) + m.Get("/hello/carl", http.NotFound) + m.Post("/hello/carl", http.NotFound) + m.Head("/hello/bob", http.NotFound) + m.Get("/hello/:name", http.NotFound) + m.Put("/hello/:name", http.NotFound) + m.Patch("/hello/:name", http.NotFound) + m.Get("/:greet/carl", http.NotFound) + m.Put("/:greet/carl", http.NotFound) + m.Delete("/:greet/:anyone", http.NotFound) for path, eMethods := range validMethodsTable { r, _ := http.NewRequest("BOGUS", path, nil) - rt.route(&C{}, httptest.NewRecorder(), r) + m.ServeHTTP(httptest.NewRecorder(), r) aMethods := <-ch if !reflect.DeepEqual(eMethods, aMethods) { t.Errorf("For %q, expected %v, got %v", path, eMethods, From 8cb07251b03147cbd281d325eb65e0d4dca82f98 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 26 Oct 2014 16:30:44 -0700 Subject: [PATCH 139/217] Fix public API f2926972236e62bb85a77c9ed50514d23fd5b294 accidentally removed web.Mux.Compile from the public API. This restores it. --- web/mux.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/mux.go b/web/mux.go index 7fafa8e..ee0d8b4 100644 --- a/web/mux.go +++ b/web/mux.go @@ -220,6 +220,6 @@ func (m *Mux) NotFound(handler interface{}) { // after all the routes have been added, and will be called automatically for // you (at some performance cost on the first request) if you do not call it // explicitly. -func (rt *router) Compile() { - rt.compile() +func (m *Mux) Compile() { + m.rt.compile() } From 89bc8315f509d641830105d7f7bb596621f8e694 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 26 Oct 2014 16:52:20 -0700 Subject: [PATCH 140/217] Move Pattern code around I think this is a bunch clearer. --- web/pattern.go | 278 ++++++------------------------------------ web/regexp_pattern.go | 145 ++++++++++++++++++++++ web/router.go | 45 ------- web/string_pattern.go | 111 +++++++++++++++++ 4 files changed, 294 insertions(+), 285 deletions(-) create mode 100644 web/regexp_pattern.go create mode 100644 web/string_pattern.go diff --git a/web/pattern.go b/web/pattern.go index be4ef34..e970788 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -1,249 +1,47 @@ package web import ( - "bytes" - "fmt" "log" "net/http" "regexp" - "regexp/syntax" - "strings" ) -type regexpPattern struct { - re *regexp.Regexp - prefix string - names []string -} - -func (p regexpPattern) Prefix() string { - return p.prefix -} -func (p regexpPattern) Match(r *http.Request, c *C) bool { - return p.match(r, c, false) -} -func (p regexpPattern) Run(r *http.Request, c *C) { - p.match(r, c, false) -} - -func (p regexpPattern) match(r *http.Request, c *C, dryrun bool) bool { - matches := p.re.FindStringSubmatch(r.URL.Path) - if matches == nil || len(matches) == 0 { - return false - } - - if c == nil || dryrun || len(matches) == 1 { - return true - } - - if c.URLParams == nil { - c.URLParams = make(map[string]string, len(matches)-1) - } - for i := 1; i < len(matches); i++ { - c.URLParams[p.names[i]] = matches[i] - } - return true -} - -func (p regexpPattern) String() string { - return fmt.Sprintf("regexpPattern(%v)", p.re) -} - -/* -I'm sorry, dear reader. I really am. - -The problem here is to take an arbitrary regular expression and: -1. return a regular expression that is just like it, but left-anchored, - preferring to return the original if possible. -2. determine a string literal prefix that all matches of this regular expression - have, much like regexp.Regexp.Prefix(). Unfortunately, Prefix() does not work - in the presence of anchors, so we need to write it ourselves. - -What this actually means is that we need to sketch on the internals of the -standard regexp library to forcefully extract the information we want. - -Unfortunately, regexp.Regexp hides a lot of its state, so our abstraction is -going to be pretty leaky. The biggest leak is that we blindly assume that all -regular expressions are perl-style, not POSIX. This is probably Mostly True, and -I think most users of the library probably won't be able to notice. -*/ -func sketchOnRegex(re *regexp.Regexp) (*regexp.Regexp, string) { - rawRe := re.String() - sRe, err := syntax.Parse(rawRe, syntax.Perl) - if err != nil { - log.Printf("WARN(web): unable to parse regexp %v as perl. "+ - "This route might behave unexpectedly.", re) - return re, "" - } - sRe = sRe.Simplify() - p, err := syntax.Compile(sRe) - if err != nil { - log.Printf("WARN(web): unable to compile regexp %v. This "+ - "route might behave unexpectedly.", re) - return re, "" - } - if p.StartCond()&syntax.EmptyBeginText == 0 { - // I hope doing this is always legal... - newRe, err := regexp.Compile(`\A` + rawRe) - if err != nil { - log.Printf("WARN(web): unable to create a left-"+ - "anchored regexp from %v. This route might "+ - "behave unexpectedly", re) - return re, "" - } - re = newRe - } - - // Run the regular expression more or less by hand :( - pc := uint32(p.Start) - atStart := true - i := &p.Inst[pc] - var buf bytes.Buffer -Sadness: - for { - switch i.Op { - case syntax.InstEmptyWidth: - if !atStart { - break Sadness - } - case syntax.InstCapture, syntax.InstNop: - // nop! - case syntax.InstRune, syntax.InstRune1, syntax.InstRuneAny, - syntax.InstRuneAnyNotNL: - - atStart = false - if len(i.Rune) != 1 || - syntax.Flags(i.Arg)&syntax.FoldCase != 0 { - break Sadness - } - buf.WriteRune(i.Rune[0]) - default: - break Sadness - } - pc = i.Out - i = &p.Inst[pc] - } - return re, buf.String() -} - -func parseRegexpPattern(re *regexp.Regexp) regexpPattern { - re, prefix := sketchOnRegex(re) - rnames := re.SubexpNames() - // We have to make our own copy since package regexp forbids us - // from scribbling over the slice returned by SubexpNames(). - names := make([]string, len(rnames)) - for i, rname := range rnames { - if rname == "" { - rname = fmt.Sprintf("$%d", i) - } - names[i] = rname - } - return regexpPattern{ - re: re, - prefix: prefix, - names: names, - } -} - -type stringPattern struct { - raw string - pats []string - literals []string - isPrefix bool -} - -func (s stringPattern) Prefix() string { - return s.literals[0] -} -func (s stringPattern) Match(r *http.Request, c *C) bool { - return s.match(r, c, true) -} -func (s stringPattern) Run(r *http.Request, c *C) { - s.match(r, c, false) -} -func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { - path := r.URL.Path - var matches map[string]string - if !dryrun && len(s.pats) > 0 { - matches = make(map[string]string, len(s.pats)) - } - for i := 0; i < len(s.pats); i++ { - sli := s.literals[i] - if !strings.HasPrefix(path, sli) { - return false - } - path = path[len(sli):] - - m := 0 - for ; m < len(path); m++ { - if path[m] == '/' { - break - } - } - if m == 0 { - // Empty strings are not matches, otherwise routes like - // "/:foo" would match the path "/" - return false - } - if !dryrun { - matches[s.pats[i]] = path[:m] - } - path = path[m:] - } - // There's exactly one more literal than pat. - if s.isPrefix { - if !strings.HasPrefix(path, s.literals[len(s.pats)]) { - return false - } - } else { - if path != s.literals[len(s.pats)] { - return false - } - } - - if c == nil || dryrun { - return true - } - - if c.URLParams == nil { - c.URLParams = matches - } else { - for k, v := range matches { - c.URLParams[k] = v - } - } - return true -} - -func (s stringPattern) String() string { - return fmt.Sprintf("stringPattern(%q, %v)", s.raw, s.isPrefix) -} - -var patternRe = regexp.MustCompile(`/:([^/]+)`) - -func parseStringPattern(s string) stringPattern { - var isPrefix bool - // Routes that end in an asterisk ("*") are prefix routes - if len(s) > 0 && s[len(s)-1] == '*' { - s = s[:len(s)-1] - isPrefix = true - } - - matches := patternRe.FindAllStringSubmatchIndex(s, -1) - pats := make([]string, len(matches)) - literals := make([]string, len(matches)+1) - n := 0 - for i, match := range matches { - a, b := match[2], match[3] - literals[i] = s[n : a-1] // Need to leave off the colon - pats[i] = s[a:b] - n = b - } - literals[len(matches)] = s[n:] - return stringPattern{ - raw: s, - pats: pats, - literals: literals, - isPrefix: isPrefix, - } +// A Pattern determines whether or not a given request matches some criteria. +// They are often used in routes, which are essentially (pattern, methodSet, +// handler) tuples. If the method and pattern match, the given handler is used. +// +// Built-in implementations of this interface are used to implement regular +// expression and string matching. +type Pattern interface { + // In practice, most real-world routes have a string prefix that can be + // used to quickly determine if a pattern is an eligible match. The + // router uses the result of this function to optimize away calls to the + // full Match function, which is likely much more expensive to compute. + // If your Pattern does not support prefixes, this function should + // return the empty string. + Prefix() string + // Returns true if the request satisfies the pattern. This function is + // free to examine both the request and the context to make this + // decision. Match should not modify either argument, and since it will + // potentially be called several times over the course of matching a + // request, it should be reasonably efficient. + Match(r *http.Request, c *C) bool + // Run the pattern on the request and context, modifying the context as + // necessary to bind URL parameters or other parsed state. + Run(r *http.Request, c *C) +} + +func parsePattern(p interface{}) Pattern { + switch p.(type) { + case Pattern: + return p.(Pattern) + case *regexp.Regexp: + return parseRegexpPattern(p.(*regexp.Regexp)) + case string: + return parseStringPattern(p.(string)) + default: + log.Fatalf("Unknown pattern type %v. Expected a web.Pattern, "+ + "regexp.Regexp, or a string.", p) + } + panic("log.Fatalf does not return") } diff --git a/web/regexp_pattern.go b/web/regexp_pattern.go new file mode 100644 index 0000000..f9f8ecc --- /dev/null +++ b/web/regexp_pattern.go @@ -0,0 +1,145 @@ +package web + +import ( + "bytes" + "fmt" + "log" + "net/http" + "regexp" + "regexp/syntax" +) + +type regexpPattern struct { + re *regexp.Regexp + prefix string + names []string +} + +func (p regexpPattern) Prefix() string { + return p.prefix +} +func (p regexpPattern) Match(r *http.Request, c *C) bool { + return p.match(r, c, false) +} +func (p regexpPattern) Run(r *http.Request, c *C) { + p.match(r, c, false) +} + +func (p regexpPattern) match(r *http.Request, c *C, dryrun bool) bool { + matches := p.re.FindStringSubmatch(r.URL.Path) + if matches == nil || len(matches) == 0 { + return false + } + + if c == nil || dryrun || len(matches) == 1 { + return true + } + + if c.URLParams == nil { + c.URLParams = make(map[string]string, len(matches)-1) + } + for i := 1; i < len(matches); i++ { + c.URLParams[p.names[i]] = matches[i] + } + return true +} + +func (p regexpPattern) String() string { + return fmt.Sprintf("regexpPattern(%v)", p.re) +} + +/* +I'm sorry, dear reader. I really am. + +The problem here is to take an arbitrary regular expression and: +1. return a regular expression that is just like it, but left-anchored, + preferring to return the original if possible. +2. determine a string literal prefix that all matches of this regular expression + have, much like regexp.Regexp.Prefix(). Unfortunately, Prefix() does not work + in the presence of anchors, so we need to write it ourselves. + +What this actually means is that we need to sketch on the internals of the +standard regexp library to forcefully extract the information we want. + +Unfortunately, regexp.Regexp hides a lot of its state, so our abstraction is +going to be pretty leaky. The biggest leak is that we blindly assume that all +regular expressions are perl-style, not POSIX. This is probably Mostly True, and +I think most users of the library probably won't be able to notice. +*/ +func sketchOnRegex(re *regexp.Regexp) (*regexp.Regexp, string) { + rawRe := re.String() + sRe, err := syntax.Parse(rawRe, syntax.Perl) + if err != nil { + log.Printf("WARN(web): unable to parse regexp %v as perl. "+ + "This route might behave unexpectedly.", re) + return re, "" + } + sRe = sRe.Simplify() + p, err := syntax.Compile(sRe) + if err != nil { + log.Printf("WARN(web): unable to compile regexp %v. This "+ + "route might behave unexpectedly.", re) + return re, "" + } + if p.StartCond()&syntax.EmptyBeginText == 0 { + // I hope doing this is always legal... + newRe, err := regexp.Compile(`\A` + rawRe) + if err != nil { + log.Printf("WARN(web): unable to create a left-"+ + "anchored regexp from %v. This route might "+ + "behave unexpectedly", re) + return re, "" + } + re = newRe + } + + // Run the regular expression more or less by hand :( + pc := uint32(p.Start) + atStart := true + i := &p.Inst[pc] + var buf bytes.Buffer +Sadness: + for { + switch i.Op { + case syntax.InstEmptyWidth: + if !atStart { + break Sadness + } + case syntax.InstCapture, syntax.InstNop: + // nop! + case syntax.InstRune, syntax.InstRune1, syntax.InstRuneAny, + syntax.InstRuneAnyNotNL: + + atStart = false + if len(i.Rune) != 1 || + syntax.Flags(i.Arg)&syntax.FoldCase != 0 { + break Sadness + } + buf.WriteRune(i.Rune[0]) + default: + break Sadness + } + pc = i.Out + i = &p.Inst[pc] + } + return re, buf.String() +} + +func parseRegexpPattern(re *regexp.Regexp) regexpPattern { + re, prefix := sketchOnRegex(re) + rnames := re.SubexpNames() + // We have to make our own copy since package regexp forbids us + // from scribbling over the slice returned by SubexpNames(). + names := make([]string, len(rnames)) + for i, rname := range rnames { + if rname == "" { + rname = fmt.Sprintf("$%d", i) + } + names[i] = rname + } + return regexpPattern{ + re: re, + prefix: prefix, + names: names, + } +} diff --git a/web/router.go b/web/router.go index f96f034..137744a 100644 --- a/web/router.go +++ b/web/router.go @@ -3,7 +3,6 @@ package web import ( "log" "net/http" - "regexp" "sort" "strings" "sync" @@ -46,9 +45,6 @@ var validMethodsMap = map[string]method{ } type route struct { - // Theory: most real world routes have a string prefix which is both - // cheap(-ish) to test against and pretty selective. And, conveniently, - // both regexes and string patterns give us this out-of-box. prefix string method method pattern Pattern @@ -61,47 +57,6 @@ type router struct { notFound Handler machine *routeMachine } - -// A Pattern determines whether or not a given request matches some criteria. -// They are often used in routes, which are essentially (pattern, methodSet, -// handler) tuples. If the method and pattern match, the given handler is used. -// -// Built-in implementations of this interface are used to implement regular -// expression and string matching. -type Pattern interface { - // In practice, most real-world routes have a string prefix that can be - // used to quickly determine if a pattern is an eligible match. The - // router uses the result of this function to optimize away calls to the - // full Match function, which is likely much more expensive to compute. - // If your Pattern does not support prefixes, this function should - // return the empty string. - Prefix() string - // Returns true if the request satisfies the pattern. This function is - // free to examine both the request and the context to make this - // decision. Match should not modify either argument, and since it will - // potentially be called several times over the course of matching a - // request, it should be reasonably efficient. - Match(r *http.Request, c *C) bool - // Run the pattern on the request and context, modifying the context as - // necessary to bind URL parameters or other parsed state. - Run(r *http.Request, c *C) -} - -func parsePattern(p interface{}) Pattern { - switch p.(type) { - case Pattern: - return p.(Pattern) - case *regexp.Regexp: - return parseRegexpPattern(p.(*regexp.Regexp)) - case string: - return parseStringPattern(p.(string)) - default: - log.Fatalf("Unknown pattern type %v. Expected a web.Pattern, "+ - "regexp.Regexp, or a string.", p) - } - panic("log.Fatalf does not return") -} - type netHTTPWrap struct { http.Handler } diff --git a/web/string_pattern.go b/web/string_pattern.go new file mode 100644 index 0000000..c6bfc3e --- /dev/null +++ b/web/string_pattern.go @@ -0,0 +1,111 @@ +package web + +import ( + "fmt" + "net/http" + "regexp" + "strings" +) + +type stringPattern struct { + raw string + pats []string + literals []string + isPrefix bool +} + +func (s stringPattern) Prefix() string { + return s.literals[0] +} +func (s stringPattern) Match(r *http.Request, c *C) bool { + return s.match(r, c, true) +} +func (s stringPattern) Run(r *http.Request, c *C) { + s.match(r, c, false) +} +func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { + path := r.URL.Path + var matches map[string]string + if !dryrun && len(s.pats) > 0 { + matches = make(map[string]string, len(s.pats)) + } + for i := 0; i < len(s.pats); i++ { + sli := s.literals[i] + if !strings.HasPrefix(path, sli) { + return false + } + path = path[len(sli):] + + m := 0 + for ; m < len(path); m++ { + if path[m] == '/' { + break + } + } + if m == 0 { + // Empty strings are not matches, otherwise routes like + // "/:foo" would match the path "/" + return false + } + if !dryrun { + matches[s.pats[i]] = path[:m] + } + path = path[m:] + } + // There's exactly one more literal than pat. + if s.isPrefix { + if !strings.HasPrefix(path, s.literals[len(s.pats)]) { + return false + } + } else { + if path != s.literals[len(s.pats)] { + return false + } + } + + if c == nil || dryrun { + return true + } + + if c.URLParams == nil { + c.URLParams = matches + } else { + for k, v := range matches { + c.URLParams[k] = v + } + } + return true +} + +func (s stringPattern) String() string { + return fmt.Sprintf("stringPattern(%q, %v)", s.raw, s.isPrefix) +} + +var patternRe = regexp.MustCompile(`/:([^/]+)`) + +func parseStringPattern(s string) stringPattern { + var isPrefix bool + // Routes that end in an asterisk ("*") are prefix routes + if len(s) > 0 && s[len(s)-1] == '*' { + s = s[:len(s)-1] + isPrefix = true + } + + matches := patternRe.FindAllStringSubmatchIndex(s, -1) + pats := make([]string, len(matches)) + literals := make([]string, len(matches)+1) + n := 0 + for i, match := range matches { + a, b := match[2], match[3] + literals[i] = s[n : a-1] // Need to leave off the colon + pats[i] = s[a:b] + n = b + } + literals[len(matches)] = s[n:] + return stringPattern{ + raw: s, + pats: pats, + literals: literals, + isPrefix: isPrefix, + } +} From a655411f3862de37b99f1e01df4ae873b1b51b92 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 26 Oct 2014 17:02:16 -0700 Subject: [PATCH 141/217] Pre-compile routes at Serve() time I originally exposed Compile() for exactly this use case, but apparently I never actually implemented this. Oops. In any event, this makes the first request a little faster (an extremely unscientific test suggests on the order of 10 microseconds). Also, if something goes terribly wrong during route compilation, it would fail before we start listening on the socket. --- serve.go | 1 + serve_appengine.go | 1 + 2 files changed, 2 insertions(+) diff --git a/serve.go b/serve.go index 0a930c3..04c245b 100644 --- a/serve.go +++ b/serve.go @@ -24,6 +24,7 @@ func Serve() { flag.Parse() } + DefaultMux.Compile() // Install our handler at the root of the standard net/http default mux. // This allows packages like expvar to continue working as expected. http.Handle("/", DefaultMux) diff --git a/serve_appengine.go b/serve_appengine.go index edf9a2d..2d39665 100644 --- a/serve_appengine.go +++ b/serve_appengine.go @@ -6,6 +6,7 @@ import "net/http" // Serve starts Goji using reasonable defaults. func Serve() { + DefaultMux.Compile() // Install our handler at the root of the standard net/http default mux. // This is required for App Engine, and also allows packages like expvar // to continue working as expected. From 1bea1bd20427b51324d447716e46dc1715c343f6 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 26 Oct 2014 17:14:04 -0700 Subject: [PATCH 142/217] Minor fixup --- web/router.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/router.go b/web/router.go index 137744a..48c3a9d 100644 --- a/web/router.go +++ b/web/router.go @@ -227,8 +227,7 @@ func (rt *router) route(c *C, w http.ResponseWriter, r *http.Request) { } func (rt *router) handleUntyped(p interface{}, m method, h interface{}) { - pat := parsePattern(p) - rt.handle(pat, m, parseHandler(h)) + rt.handle(parsePattern(p), m, parseHandler(h)) } func (rt *router) handle(p Pattern, m method, h Handler) { From b12ae7fd2234d61f2ae0c3f9d048e0c57b36336d Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 26 Oct 2014 17:27:44 -0700 Subject: [PATCH 143/217] Move bytecode router code around --- web/{fast_router.go => bytecode_compiler.go} | 0 web/bytecode_runner.go | 88 ++++++++++++++++++++ web/router.go | 85 ------------------- 3 files changed, 88 insertions(+), 85 deletions(-) rename web/{fast_router.go => bytecode_compiler.go} (100%) create mode 100644 web/bytecode_runner.go diff --git a/web/fast_router.go b/web/bytecode_compiler.go similarity index 100% rename from web/fast_router.go rename to web/bytecode_compiler.go diff --git a/web/bytecode_runner.go b/web/bytecode_runner.go new file mode 100644 index 0000000..9758401 --- /dev/null +++ b/web/bytecode_runner.go @@ -0,0 +1,88 @@ +package web + +import "net/http" + +type routeMachine struct { + sm stateMachine + routes []route +} + +func matchRoute(route route, m method, ms *method, r *http.Request, c *C) bool { + if !route.pattern.Match(r, c) { + return false + } + *ms |= route.method + + if route.method&m != 0 { + route.pattern.Run(r, c) + return true + } + return false +} + +func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (method, bool) { + m := httpMethod(r.Method) + var methods method + p := r.URL.Path + + if len(rm.sm) == 0 { + return methods, false + } + + var i int + for { + sm := rm.sm[i].mode + if sm&smSetCursor != 0 { + si := rm.sm[i].i + p = r.URL.Path[si:] + i++ + continue + } + + length := int(sm & smLengthMask) + match := false + if length <= len(p) { + bs := rm.sm[i].bs + switch length { + case 3: + if p[2] != bs[2] { + break + } + fallthrough + case 2: + if p[1] != bs[1] { + break + } + fallthrough + case 1: + if p[0] != bs[0] { + break + } + fallthrough + case 0: + p = p[length:] + match = true + } + } + + if match && sm&smRoute != 0 { + si := rm.sm[i].i + if matchRoute(rm.routes[si], m, &methods, r, c) { + rm.routes[si].handler.ServeHTTPC(*c, w, r) + return 0, true + } + i++ + } else if (match && sm&smJumpOnMatch != 0) || + (!match && sm&smJumpOnMatch == 0) { + + if sm&smFail != 0 { + return methods, false + } + i = int(rm.sm[i].i) + } else { + i++ + } + } + + return methods, false +} diff --git a/web/router.go b/web/router.go index 48c3a9d..c0f03e5 100644 --- a/web/router.go +++ b/web/router.go @@ -96,91 +96,6 @@ func httpMethod(mname string) method { return mIDK } -type routeMachine struct { - sm stateMachine - routes []route -} - -func matchRoute(route route, m method, ms *method, r *http.Request, c *C) bool { - if !route.pattern.Match(r, c) { - return false - } - *ms |= route.method - - if route.method&m != 0 { - route.pattern.Run(r, c) - return true - } - return false -} - -func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (method, bool) { - m := httpMethod(r.Method) - var methods method - p := r.URL.Path - - if len(rm.sm) == 0 { - return methods, false - } - - var i int - for { - sm := rm.sm[i].mode - if sm&smSetCursor != 0 { - si := rm.sm[i].i - p = r.URL.Path[si:] - i++ - continue - } - - length := int(sm & smLengthMask) - match := false - if length <= len(p) { - bs := rm.sm[i].bs - switch length { - case 3: - if p[2] != bs[2] { - break - } - fallthrough - case 2: - if p[1] != bs[1] { - break - } - fallthrough - case 1: - if p[0] != bs[0] { - break - } - fallthrough - case 0: - p = p[length:] - match = true - } - } - - if match && sm&smRoute != 0 { - si := rm.sm[i].i - if matchRoute(rm.routes[si], m, &methods, r, c) { - rm.routes[si].handler.ServeHTTPC(*c, w, r) - return 0, true - } - i++ - } else if (match && sm&smJumpOnMatch != 0) || - (!match && sm&smJumpOnMatch == 0) { - - if sm&smFail != 0 { - return methods, false - } - i = int(rm.sm[i].i) - } else { - i++ - } - } - - return methods, false -} - func (rt *router) compile() *routeMachine { rt.lock.Lock() defer rt.lock.Unlock() From f255c52789a1c4f255b5533eead00766369c1611 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 26 Oct 2014 17:44:00 -0700 Subject: [PATCH 144/217] Change the router interface slightly Make the bytecode runner return the route that we're going to use. It's up to the router itself to dispatch to that route. Besides feeling a teensy bit cleaner, this refactoring is to prepare for a "Router" middleware, which will allow application developers to control when in the middleware stack routing occurs. --- web/bytecode_runner.go | 11 +++++------ web/router.go | 5 +++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/web/bytecode_runner.go b/web/bytecode_runner.go index 9758401..706490a 100644 --- a/web/bytecode_runner.go +++ b/web/bytecode_runner.go @@ -20,13 +20,13 @@ func matchRoute(route route, m method, ms *method, r *http.Request, c *C) bool { return false } -func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (method, bool) { +func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (method, *route) { m := httpMethod(r.Method) var methods method p := r.URL.Path if len(rm.sm) == 0 { - return methods, false + return methods, nil } var i int @@ -68,15 +68,14 @@ func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (meth if match && sm&smRoute != 0 { si := rm.sm[i].i if matchRoute(rm.routes[si], m, &methods, r, c) { - rm.routes[si].handler.ServeHTTPC(*c, w, r) - return 0, true + return 0, &rm.routes[si] } i++ } else if (match && sm&smJumpOnMatch != 0) || (!match && sm&smJumpOnMatch == 0) { if sm&smFail != 0 { - return methods, false + return methods, nil } i = int(rm.sm[i].i) } else { @@ -84,5 +83,5 @@ func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (meth } } - return methods, false + return methods, nil } diff --git a/web/router.go b/web/router.go index c0f03e5..632d24e 100644 --- a/web/router.go +++ b/web/router.go @@ -113,8 +113,9 @@ func (rt *router) route(c *C, w http.ResponseWriter, r *http.Request) { rm = rt.compile() } - methods, ok := rm.route(c, w, r) - if ok { + methods, route := rm.route(c, w, r) + if route != nil { + route.handler.ServeHTTPC(*c, w, r) return } From ac6efd8f69cec8b916082cf7e45d0d515030e03e Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Wed, 29 Oct 2014 22:32:29 -0700 Subject: [PATCH 145/217] Add a note about outgrowing Logger This is a common enough question that it's worth calling out explicitly. --- web/middleware/logger.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web/middleware/logger.go b/web/middleware/logger.go index fce822e..e0fb9aa 100644 --- a/web/middleware/logger.go +++ b/web/middleware/logger.go @@ -16,6 +16,13 @@ import ( // print in color, otherwise it will print in black and white. // // Logger prints a request ID if one is provided. +// +// Logger has been designed explicitly to be Good Enough for use in small +// applications and for people just getting started with Goji. It is expected +// that applications will eventually outgrow this middleware and replace it with +// a custom request logger, such as one that produces machine-parseable output, +// outputs logs to a different service (e.g., syslog), or formats lines like +// those printed elsewhere in the application. func Logger(c *web.C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { reqID := GetReqID(*c) From 1a390aba1c7c591e2c0c75be43eb830091b7d6f7 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 1 Nov 2014 14:59:43 -0700 Subject: [PATCH 146/217] Allow ".,;" as string pattern "break chars" This allows you to match "/a/cat.gif" with patterns like "/a/:b.:c". Thanks to @Minecrell for an early patch implementing this functionality. Fixes #75. Fixes #48. --- web/pattern_test.go | 18 ++++++++++++++++++ web/string_pattern.go | 20 ++++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/web/pattern_test.go b/web/pattern_test.go index 6b2575f..3794270 100644 --- a/web/pattern_test.go +++ b/web/pattern_test.go @@ -93,6 +93,9 @@ var patternTests = []struct { pt("/hello/world", true, map[string]string{ "name": "world", }), + pt("/hello/my.world;wow", true, map[string]string{ + "name": "my.world;wow", + }), pt("/hell", false, nil), pt("/hello/", false, nil), pt("/hello/my/love", false, nil), @@ -107,6 +110,21 @@ var patternTests = []struct { pt("/a//b/", false, nil), pt("/a/1/b/2/3", false, nil), }}, + {parseStringPattern("/a/:b.:c"), + "/a/", []patternTest{ + pt("/a/cat.gif", true, map[string]string{ + "b": "cat", + "c": "gif", + }), + pt("/a/cat.tar.gz", true, map[string]string{ + "b": "cat", + "c": "tar.gz", + }), + pt("/a", false, nil), + pt("/a/cat", false, nil), + pt("/a/cat/gif", false, nil), + pt("/a/cat.", false, nil), + }}, // String prefix tests {parseStringPattern("/user/:user*"), diff --git a/web/string_pattern.go b/web/string_pattern.go index c6bfc3e..1dcb3c7 100644 --- a/web/string_pattern.go +++ b/web/string_pattern.go @@ -7,9 +7,11 @@ import ( "strings" ) +// stringPattern is a struct describing type stringPattern struct { raw string pats []string + breaks []byte literals []string isPrefix bool } @@ -37,8 +39,9 @@ func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { path = path[len(sli):] m := 0 + bc := s.breaks[i] for ; m < len(path); m++ { - if path[m] == '/' { + if path[m] == bc { break } } @@ -81,7 +84,13 @@ func (s stringPattern) String() string { return fmt.Sprintf("stringPattern(%q, %v)", s.raw, s.isPrefix) } -var patternRe = regexp.MustCompile(`/:([^/]+)`) +// "Break characters" are characters that can end patterns. They are not allowed +// to appear in pattern names. "/" was chosen because it is the standard path +// separator, and "." was chosen because it often delimits file extensions. ";" +// and "," were chosen because Section 3.3 of RFC 3986 suggests their use. +const bc = "/.;," + +var patternRe = regexp.MustCompile(`[` + bc + `]:([^` + bc + `]+)`) func parseStringPattern(s string) stringPattern { var isPrefix bool @@ -93,18 +102,25 @@ func parseStringPattern(s string) stringPattern { matches := patternRe.FindAllStringSubmatchIndex(s, -1) pats := make([]string, len(matches)) + breaks := make([]byte, len(matches)) literals := make([]string, len(matches)+1) n := 0 for i, match := range matches { a, b := match[2], match[3] literals[i] = s[n : a-1] // Need to leave off the colon pats[i] = s[a:b] + if b == len(s) { + breaks[i] = '/' + } else { + breaks[i] = s[b] + } n = b } literals[len(matches)] = s[n:] return stringPattern{ raw: s, pats: pats, + breaks: breaks, literals: literals, isPrefix: isPrefix, } From 2fe5c3ee437887e03c7a068e2feb0a21b89886c6 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 1 Nov 2014 15:46:02 -0700 Subject: [PATCH 147/217] Change semantics of wildcard patterns This is a breaking API change that changes how wildcard patterns are treated. In particular, wildcards are no longer allowed to appear at arbitrary places in the URL, and are only allowed to appear immediately after a path separator. This change effectively changes the wildcard sigil from "*" to "/*". Users who use wildcard routes like "/hello*" will have to switch to regular expression based routes to preserve the old semantics. The motivation for this change is that it allows the router to publish a special "tail" key which represents the unmatched portion of the URL. This is placed into URLParams under the key "*", and includes a leading "/" to make it easier to write sub-routers. --- example/main.go | 8 ++++---- web/mux.go | 13 ++++++++----- web/pattern_test.go | 24 ++++++++---------------- web/router_test.go | 2 +- web/string_pattern.go | 36 +++++++++++++++++++++--------------- 5 files changed, 42 insertions(+), 41 deletions(-) diff --git a/example/main.go b/example/main.go index 9315af6..ccfae49 100644 --- a/example/main.go +++ b/example/main.go @@ -39,10 +39,10 @@ func main() { // can put them wherever you like. goji.Use(PlainText) - // If the last character of a pattern is an asterisk, the path is - // treated as a prefix, and can be used to implement sub-routes. - // Sub-routes can be used to set custom middleware on sub-applications. - // Goji's interfaces are completely composable. + // If the patterns ends with "/*", the path is treated as a prefix, and + // can be used to implement sub-routes. Sub-routes can be used to set + // custom middleware on sub-applications. Goji's interfaces are + // completely composable. admin := web.New() goji.Handle("/admin/*", admin) admin.Use(SuperSecure) diff --git a/web/mux.go b/web/mux.go index ee0d8b4..0004bae 100644 --- a/web/mux.go +++ b/web/mux.go @@ -35,11 +35,14 @@ and handler. Pattern must be one of the following types: - a path segment starting with with a colon will match any string placed at that position. e.g., "/:name" will match "/carl", binding "name" to "carl". - - a pattern ending with an asterisk will match any prefix of - that route. For instance, "/admin/*" will match "/admin/" and - "/admin/secret/lair". This is similar to Sinatra's wildcard, - but may only appear at the very end of the string and is - therefore significantly less powerful. + - a pattern ending with "/*" will match any route with that + prefix. For instance, the pattern "/u/:name/*" will match + "/u/carl/" and "/u/carl/projects/123", but not "/u/carl" + (because there is no trailing slash). In addition to any names + bound in the pattern, the special name "*" is bound to the + unmatched tail of the match, but including the leading "/". So + for the two matching examples above, "*" would be bound to "/" + and "/projects/123" respectively. - regexp.Regexp. The library assumes that it is a Perl-style regexp that is anchored on the left (i.e., the beginning of the string). If your regexp is not anchored on the left, a hopefully-identical diff --git a/web/pattern_test.go b/web/pattern_test.go index 3794270..7fb3c7a 100644 --- a/web/pattern_test.go +++ b/web/pattern_test.go @@ -127,37 +127,29 @@ var patternTests = []struct { }}, // String prefix tests - {parseStringPattern("/user/:user*"), + {parseStringPattern("/user/:user/*"), "/user/", []patternTest{ - pt("/user/bob", true, map[string]string{ + pt("/user/bob/", true, map[string]string{ "user": "bob", + "*": "/", }), pt("/user/bob/friends/123", true, map[string]string{ "user": "bob", - }), - pt("/user/", false, nil), - pt("/user//", false, nil), - }}, - {parseStringPattern("/user/:user/*"), - "/user/", []patternTest{ - pt("/user/bob/friends/123", true, map[string]string{ - "user": "bob", + "*": "/friends/123", }), pt("/user/bob", false, nil), pt("/user/", false, nil), pt("/user//", false, nil), }}, - {parseStringPattern("/user/:user/friends*"), + {parseStringPattern("/user/:user/friends/*"), "/user/", []patternTest{ - pt("/user/bob/friends", true, map[string]string{ + pt("/user/bob/friends/", true, map[string]string{ "user": "bob", + "*": "/", }), pt("/user/bob/friends/123", true, map[string]string{ "user": "bob", - }), - // This is a little unfortunate - pt("/user/bob/friends123", true, map[string]string{ - "user": "bob", + "*": "/123", }), pt("/user/bob/enemies", false, nil), }}, diff --git a/web/router_test.go b/web/router_test.go index b4c9730..b80862d 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -244,7 +244,7 @@ func TestPrefix(t *testing.T) { m := New() ch := make(chan string, 1) - m.Handle("/hello*", func(w http.ResponseWriter, r *http.Request) { + m.Handle("/hello/*", func(w http.ResponseWriter, r *http.Request) { ch <- r.URL.Path }) diff --git a/web/string_pattern.go b/web/string_pattern.go index 1dcb3c7..197c7c9 100644 --- a/web/string_pattern.go +++ b/web/string_pattern.go @@ -13,7 +13,7 @@ type stringPattern struct { pats []string breaks []byte literals []string - isPrefix bool + wildcard bool } func (s stringPattern) Prefix() string { @@ -28,8 +28,12 @@ func (s stringPattern) Run(r *http.Request, c *C) { func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { path := r.URL.Path var matches map[string]string - if !dryrun && len(s.pats) > 0 { - matches = make(map[string]string, len(s.pats)) + if !dryrun { + if s.wildcard { + matches = make(map[string]string, len(s.pats)+1) + } else if len(s.pats) != 0 { + matches = make(map[string]string, len(s.pats)) + } } for i := 0; i < len(s.pats); i++ { sli := s.literals[i] @@ -56,14 +60,16 @@ func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { path = path[m:] } // There's exactly one more literal than pat. - if s.isPrefix { - if !strings.HasPrefix(path, s.literals[len(s.pats)]) { + tail := s.literals[len(s.pats)] + if s.wildcard { + if !strings.HasPrefix(path, tail) { return false } - } else { - if path != s.literals[len(s.pats)] { - return false + if !dryrun { + matches["*"] = path[len(tail)-1:] } + } else if path != tail { + return false } if c == nil || dryrun { @@ -81,7 +87,7 @@ func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { } func (s stringPattern) String() string { - return fmt.Sprintf("stringPattern(%q, %v)", s.raw, s.isPrefix) + return fmt.Sprintf("stringPattern(%q)", s.raw) } // "Break characters" are characters that can end patterns. They are not allowed @@ -93,11 +99,11 @@ const bc = "/.;," var patternRe = regexp.MustCompile(`[` + bc + `]:([^` + bc + `]+)`) func parseStringPattern(s string) stringPattern { - var isPrefix bool - // Routes that end in an asterisk ("*") are prefix routes - if len(s) > 0 && s[len(s)-1] == '*' { + raw := s + var wildcard bool + if strings.HasSuffix(s, "/*") { s = s[:len(s)-1] - isPrefix = true + wildcard = true } matches := patternRe.FindAllStringSubmatchIndex(s, -1) @@ -118,10 +124,10 @@ func parseStringPattern(s string) stringPattern { } literals[len(matches)] = s[n:] return stringPattern{ - raw: s, + raw: raw, pats: pats, breaks: breaks, literals: literals, - isPrefix: isPrefix, + wildcard: wildcard, } } From 69ab4d722ea1317d12cd1bce53887182c04cccae Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 1 Nov 2014 16:17:05 -0700 Subject: [PATCH 148/217] SubRouter middleware This middleware makes it much easier to write sub-routes, allowing you to make the sub-router ignorant of its parent router's matched prefix. The example app has also been modified to use this functionality for its admin pages. Fixes #65. --- example/main.go | 23 +++++++++++++++-------- web/middleware/subrouter.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 web/middleware/subrouter.go diff --git a/example/main.go b/example/main.go index ccfae49..8d6b634 100644 --- a/example/main.go +++ b/example/main.go @@ -18,6 +18,7 @@ import ( "github.com/goji/param" "github.com/zenazn/goji" "github.com/zenazn/goji/web" + "github.com/zenazn/goji/web/middleware" ) // Note: the code below cuts a lot of corners to make the example app simple. @@ -40,22 +41,28 @@ func main() { goji.Use(PlainText) // If the patterns ends with "/*", the path is treated as a prefix, and - // can be used to implement sub-routes. Sub-routes can be used to set - // custom middleware on sub-applications. Goji's interfaces are - // completely composable. + // can be used to implement sub-routes. admin := web.New() goji.Handle("/admin/*", admin) + + // The standard SubRouter middleware helps make writing sub-routers + // easy. Ordinarily, Goji does not manipulate the request's URL.Path, + // meaning you'd have to repeat "/admin/" in each of the following + // routes. This middleware allows you to cut down on the repetition by + // eliminating the shared, already-matched prefix. + admin.Use(middleware.SubRouter) + // You can also easily attach extra middleware to sub-routers that are + // not present on the parent router. This one, for instance, presents a + // password prompt to users of the admin endpoints. admin.Use(SuperSecure) + admin.Get("/", AdminRoot) + admin.Get("/finances", AdminFinances) + // Goji's routing, like Sinatra's, is exact: no effort is made to // normalize trailing slashes. goji.Get("/admin", http.RedirectHandler("/admin/", 301)) - // Set up admin routes. Note that sub-routes do *not* mutate the path in - // any way, so we need to supply full ("/admin/" prefixed) paths. - admin.Get("/admin/", AdminRoot) - admin.Get("/admin/finances", AdminFinances) - // Use a custom 404 handler goji.NotFound(NotFound) diff --git a/web/middleware/subrouter.go b/web/middleware/subrouter.go new file mode 100644 index 0000000..60de471 --- /dev/null +++ b/web/middleware/subrouter.go @@ -0,0 +1,36 @@ +package middleware + +import ( + "net/http" + + "github.com/zenazn/goji/web" +) + +type subrouter struct { + c *web.C + h http.Handler +} + +func (s subrouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if s.c.URLParams != nil { + if path, ok := s.c.URLParams["*"]; ok { + oldpath := r.URL.Path + r.URL.Path = path + defer func() { + r.URL.Path = oldpath + }() + } + } + s.h.ServeHTTP(w, r) +} + +// SubRouter is a helper middleware that makes writing sub-routers easier. +// +// If you register a sub-router under a key like "/admin/*", Goji's router will +// automatically set c.URLParams["*"] to the unmatched path suffix. This +// middleware will help you set the request URL's Path to this unmatched suffix, +// allowing you to write sub-routers with no knowledge of what routes the parent +// router matches. +func SubRouter(c *web.C, h http.Handler) http.Handler { + return subrouter{c, h} +} From 9556e1736d72993303989f72b3fef2450d121d92 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 1 Nov 2014 17:06:20 -0700 Subject: [PATCH 149/217] Force use of fallback chanpool on App Engine This is to work around a bug where App Engine, which as of this writing runs Go 1.2, builds code which uses the "go1.3" build tag. See #67. This bug is being tracked at: https://code.google.com/p/googleappengine/issues/detail?id=11346 --- web/chanpool.go | 2 +- web/cpool.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/chanpool.go b/web/chanpool.go index fbe2977..26b675e 100644 --- a/web/chanpool.go +++ b/web/chanpool.go @@ -1,4 +1,4 @@ -// +build !go1.3 +// +build !go1.3 appengine package web diff --git a/web/cpool.go b/web/cpool.go index 59f8764..a95333c 100644 --- a/web/cpool.go +++ b/web/cpool.go @@ -1,4 +1,4 @@ -// +build go1.3 +// +build go1.3,!appengine package web From 60d46a21e36a7d7ea7e9bb96184683a6e72c0562 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 2 Nov 2014 13:00:14 -0800 Subject: [PATCH 150/217] Fix typo in a test error message --- web/func_equal_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/func_equal_test.go b/web/func_equal_test.go index daf8d9a..9080a5a 100644 --- a/web/func_equal_test.go +++ b/web/func_equal_test.go @@ -79,6 +79,6 @@ func TestFuncEqual(t *testing.T) { t.Errorf("h and j should not have been equal") } if !funcEqual(a, k) { - t.Errorf("a and k should not have been equal") + t.Errorf("a and k should have been equal") } } From 17227c10c48acd3a82984f3009f0cd436ac2b72b Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 2 Nov 2014 13:01:32 -0800 Subject: [PATCH 151/217] Improve test coverage slightly --- web/router_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/web/router_test.go b/web/router_test.go index b80862d..44dabf5 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -306,4 +306,21 @@ func TestValidMethods(t *testing.T) { aMethods) } } + + // This should also work when c.Env has already been initalized + m.Use(func(c *C, h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.Env = make(map[string]interface{}) + h.ServeHTTP(w, r) + }) + }) + for path, eMethods := range validMethodsTable { + r, _ := http.NewRequest("BOGUS", path, nil) + m.ServeHTTP(httptest.NewRecorder(), r) + aMethods := <-ch + if !reflect.DeepEqual(eMethods, aMethods) { + t.Errorf("For %q, expected %v, got %v", path, eMethods, + aMethods) + } + } } From dde29893cc702abb56381731fc92f97d75b2f9ca Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 2 Nov 2014 13:22:57 -0800 Subject: [PATCH 152/217] Simplify a logical XOR --- web/bytecode_runner.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/web/bytecode_runner.go b/web/bytecode_runner.go index 706490a..5fcf443 100644 --- a/web/bytecode_runner.go +++ b/web/bytecode_runner.go @@ -71,9 +71,7 @@ func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (meth return 0, &rm.routes[si] } i++ - } else if (match && sm&smJumpOnMatch != 0) || - (!match && sm&smJumpOnMatch == 0) { - + } else if match != (sm&smJumpOnMatch == 0) { if sm&smFail != 0 { return methods, nil } From 23ec88ea5a560383b8b3ed7a5ef18045a36f05f9 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 3 Nov 2014 11:29:51 -0800 Subject: [PATCH 153/217] Fix bug with string pattern matching with "." Previously, a route like "/:a.png" would match "/foo/bar/baz.png". This was incorrect, as all matches should be restricted to path segments (or smaller, as dictated by a break character). This bug was introduced in 1a390aba1c7c591e2c0c75be43eb830091b7d6f7. Fixes #75. --- web/pattern_test.go | 1 + web/string_pattern.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/web/pattern_test.go b/web/pattern_test.go index 7fb3c7a..8e8af29 100644 --- a/web/pattern_test.go +++ b/web/pattern_test.go @@ -124,6 +124,7 @@ var patternTests = []struct { pt("/a/cat", false, nil), pt("/a/cat/gif", false, nil), pt("/a/cat.", false, nil), + pt("/a/cat/dog.gif", false, nil), }}, // String prefix tests diff --git a/web/string_pattern.go b/web/string_pattern.go index 197c7c9..f68e044 100644 --- a/web/string_pattern.go +++ b/web/string_pattern.go @@ -45,7 +45,7 @@ func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { m := 0 bc := s.breaks[i] for ; m < len(path); m++ { - if path[m] == bc { + if path[m] == bc || path[m] == '/' { break } } From e1643f8e1cc4e348fb69f1f44907088b5bf1f2ce Mon Sep 17 00:00:00 2001 From: Marcus Redivo Date: Tue, 4 Nov 2014 11:56:32 -0800 Subject: [PATCH 154/217] Remove signal.go logging; log from hook functions instead. --- graceful/signal.go | 4 +--- serve.go | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/graceful/signal.go b/graceful/signal.go index 5c2ab15..4d9f0ae 100644 --- a/graceful/signal.go +++ b/graceful/signal.go @@ -1,7 +1,6 @@ package graceful import ( - "log" "os" "os/signal" "sync" @@ -90,8 +89,7 @@ func PostHook(f func()) { } func waitForSignal() { - sig := <-sigchan - log.Printf("Received %v, gracefully shutting down!", sig) + <-sigchan hookLock.Lock() defer hookLock.Unlock() diff --git a/serve.go b/serve.go index 04c245b..cb4e405 100644 --- a/serve.go +++ b/serve.go @@ -34,6 +34,8 @@ func Serve() { graceful.HandleSignals() bind.Ready() + graceful.PreHook(func() { log.Printf("Goji received signal, gracefully stopping") }) + graceful.PostHook(func() { log.Printf("Goji stopped") }) err := graceful.Serve(listener, http.DefaultServeMux) From a29ff223dc9e977f3dcfcf69508e0bd5b7cb068b Mon Sep 17 00:00:00 2001 From: SatoShun Date: Wed, 5 Nov 2014 13:41:20 +0000 Subject: [PATCH 155/217] Use range syntax --- web/string_pattern.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/string_pattern.go b/web/string_pattern.go index f68e044..9e9945a 100644 --- a/web/string_pattern.go +++ b/web/string_pattern.go @@ -35,7 +35,7 @@ func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { matches = make(map[string]string, len(s.pats)) } } - for i := 0; i < len(s.pats); i++ { + for i, pat := range s.pats { sli := s.literals[i] if !strings.HasPrefix(path, sli) { return false @@ -55,7 +55,7 @@ func (s stringPattern) match(r *http.Request, c *C, dryrun bool) bool { return false } if !dryrun { - matches[s.pats[i]] = path[:m] + matches[pat] = path[:m] } path = path[m:] } From 9a8e832951bb06c5b52393c237be7680d8ed7f44 Mon Sep 17 00:00:00 2001 From: SatoShun Date: Fri, 7 Nov 2014 13:03:36 +0000 Subject: [PATCH 156/217] use direct result of switch type assertion --- web/middleware.go | 7 +++---- web/pattern.go | 8 ++++---- web/router.go | 8 +++----- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/web/middleware.go b/web/middleware.go index db766ad..21e000b 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -52,14 +52,13 @@ func (s *cStack) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { func (m *mStack) appendLayer(fn interface{}) { ml := mLayer{orig: fn} - switch fn.(type) { + switch f := fn.(type) { case func(http.Handler) http.Handler: - unwrapped := fn.(func(http.Handler) http.Handler) ml.fn = func(c *C, h http.Handler) http.Handler { - return unwrapped(h) + return f(h) } case func(*C, http.Handler) http.Handler: - ml.fn = fn.(func(*C, http.Handler) http.Handler) + ml.fn = f default: log.Fatalf(`Unknown middleware type %v. Expected a function `+ `with signature "func(http.Handler) http.Handler" or `+ diff --git a/web/pattern.go b/web/pattern.go index e970788..0d27c51 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -32,13 +32,13 @@ type Pattern interface { } func parsePattern(p interface{}) Pattern { - switch p.(type) { + switch v := p.(type) { case Pattern: - return p.(Pattern) + return v case *regexp.Regexp: - return parseRegexpPattern(p.(*regexp.Regexp)) + return parseRegexpPattern(v) case string: - return parseStringPattern(p.(string)) + return parseStringPattern(v) default: log.Fatalf("Unknown pattern type %v. Expected a web.Pattern, "+ "regexp.Regexp, or a string.", p) diff --git a/web/router.go b/web/router.go index 632d24e..d4b88d5 100644 --- a/web/router.go +++ b/web/router.go @@ -69,16 +69,14 @@ func (h netHTTPWrap) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { } func parseHandler(h interface{}) Handler { - switch h.(type) { + switch f := h.(type) { case Handler: - return h.(Handler) + return f case http.Handler: - return netHTTPWrap{h.(http.Handler)} + return netHTTPWrap{f} case func(c C, w http.ResponseWriter, r *http.Request): - f := h.(func(c C, w http.ResponseWriter, r *http.Request)) return HandlerFunc(f) case func(w http.ResponseWriter, r *http.Request): - f := h.(func(w http.ResponseWriter, r *http.Request)) return netHTTPWrap{http.HandlerFunc(f)} default: log.Fatalf("Unknown handler type %v. Expected a web.Handler, "+ From 2b08b90ead97c2f9d59b0d29155f3014b903786c Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Nov 2014 14:08:19 -0800 Subject: [PATCH 157/217] Use new "cover" path The official "cover" package has moved, and Go 1.4 will start enforcing canonical imports. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8d7d11f..e991787 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ go: install: - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v - - go get code.google.com/p/go.tools/cmd/cover + - go get golang.org/x/tools/cmd/cover - go build -v ./... script: - go test -v -cover ./... From 8942c8c5667c6c086ac5c7e7144e5c73ab7ff4c4 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 23 Nov 2014 14:12:38 -0800 Subject: [PATCH 158/217] Fix build This is pretty frustrating: it doesn't look like it's going to be easy to get the "cover" tool to work across both 1.3 and 1.4, since they aren't able to agree on where to install the tool from. I'm sure there's a way to work around this, but in the meantime let's just disable coverage testing entirely. --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e991787..493b209 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ go: install: - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v - - go get golang.org/x/tools/cmd/cover - go build -v ./... script: - - go test -v -cover ./... + - go test -v ./... From c35d6c02ad4a1260e50276566cd6392f9803bd3e Mon Sep 17 00:00:00 2001 From: Marcus Redivo Date: Tue, 25 Nov 2014 09:55:34 -0800 Subject: [PATCH 159/217] Prevent shutdown race, and refuse new requests. --- graceful/net.go | 17 +++++++++++++---- graceful/signal.go | 9 +++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/graceful/net.go b/graceful/net.go index e667d05..784003d 100644 --- a/graceful/net.go +++ b/graceful/net.go @@ -58,10 +58,19 @@ func WrapConn(c net.Conn) net.Conn { return nil } - wg.Add(1) - return &conn{ - Conn: c, - id: atomic.AddUint64(&idleSet.id, 1), + // Avoid race with termination code. + wgLock.Lock() + defer wgLock.Unlock() + + // Determine whether the app is shutting down. + if acceptingRequests { + wg.Add(1) + return &conn{ + Conn: c, + id: atomic.AddUint64(&idleSet.id, 1), + } + } else { + return nil } } diff --git a/graceful/signal.go b/graceful/signal.go index 4d9f0ae..82a13d6 100644 --- a/graceful/signal.go +++ b/graceful/signal.go @@ -14,9 +14,13 @@ var kill = make(chan struct{}) // closed once all the posthooks have been called. var wait = make(chan struct{}) +// Whether new requests should be accepted. When false, new requests are refused. +var acceptingRequests bool = true + // This is the WaitGroup that indicates when all the connections have gracefully // shut down. var wg sync.WaitGroup +var wgLock sync.Mutex // This lock protects the list of pre- and post- hooks below. var hookLock sync.Mutex @@ -91,6 +95,11 @@ func PostHook(f func()) { func waitForSignal() { <-sigchan + // Prevent servicing of any new requests. + wgLock.Lock() + acceptingRequests = false + wgLock.Unlock() + hookLock.Lock() defer hookLock.Unlock() From 2ca8864e18b41cf0cb4ad3beaf317257befed33e Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 7 Dec 2014 15:19:13 -0800 Subject: [PATCH 160/217] s/util/mutil/ "util" is a really bad name for a package since it isn't very descriptive and so often collides with other names. Unfortunately, this is a breaking change, but it's both very easy to fix and perhaps more importantly also better to do now than later. --- web/middleware/logger.go | 6 +++--- web/mutil/mutil.go | 3 +++ web/{util => mutil}/writer_proxy.go | 6 +++--- web/util/util.go | 3 --- 4 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 web/mutil/mutil.go rename web/{util => mutil}/writer_proxy.go (95%) delete mode 100644 web/util/util.go diff --git a/web/middleware/logger.go b/web/middleware/logger.go index e0fb9aa..8bbcac8 100644 --- a/web/middleware/logger.go +++ b/web/middleware/logger.go @@ -7,7 +7,7 @@ import ( "time" "github.com/zenazn/goji/web" - "github.com/zenazn/goji/web/util" + "github.com/zenazn/goji/web/mutil" ) // Logger is a middleware that logs the start and end of each request, along @@ -29,7 +29,7 @@ func Logger(c *web.C, h http.Handler) http.Handler { printStart(reqID, r) - lw := util.WrapWriter(w) + lw := mutil.WrapWriter(w) t1 := time.Now() h.ServeHTTP(lw, r) @@ -60,7 +60,7 @@ func printStart(reqID string, r *http.Request) { log.Print(buf.String()) } -func printEnd(reqID string, w util.WriterProxy, dt time.Duration) { +func printEnd(reqID string, w mutil.WriterProxy, dt time.Duration) { var buf bytes.Buffer if reqID != "" { diff --git a/web/mutil/mutil.go b/web/mutil/mutil.go new file mode 100644 index 0000000..e8d5b28 --- /dev/null +++ b/web/mutil/mutil.go @@ -0,0 +1,3 @@ +// Package mutil contains various functions that are helpful when writing http +// middleware. +package mutil diff --git a/web/util/writer_proxy.go b/web/mutil/writer_proxy.go similarity index 95% rename from web/util/writer_proxy.go rename to web/mutil/writer_proxy.go index 4406a3c..d572812 100644 --- a/web/util/writer_proxy.go +++ b/web/mutil/writer_proxy.go @@ -1,4 +1,4 @@ -package util +package mutil import ( "bufio" @@ -27,8 +27,8 @@ type WriterProxy interface { Unwrap() http.ResponseWriter } -// WrapWriter wraps an http.ResponseWriter into a proxy that allows you to hook -// into various parts of the response process. +// WrapWriter wraps an http.ResponseWriter, returning a proxy that allows you to +// hook into various parts of the response process. func WrapWriter(w http.ResponseWriter) WriterProxy { _, cn := w.(http.CloseNotifier) _, fl := w.(http.Flusher) diff --git a/web/util/util.go b/web/util/util.go deleted file mode 100644 index aa9dda8..0000000 --- a/web/util/util.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package util contains various functions that are helpful when writing http -// middleware. -package util From 83e1241fde7a0a387782f57cd6a878a8a05a36e2 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 7 Dec 2014 16:35:41 -0800 Subject: [PATCH 161/217] Make ParsePattern public The goal here is to make it easy for people to write thin wrappers around the built-in Patterns. --- web/mux.go | 24 ++++-------------------- web/pattern.go | 39 +++++++++++++++++++++++++++++++++++---- web/router.go | 2 +- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/web/mux.go b/web/mux.go index 0004bae..3908de5 100644 --- a/web/mux.go +++ b/web/mux.go @@ -28,27 +28,11 @@ A middleware (the untyped parameter in Use() and Insert()) must be one of the following types: - func(http.Handler) http.Handler - func(c *web.C, http.Handler) http.Handler + All of the route-adding functions on Mux take two untyped parameters: pattern -and handler. Pattern must be one of the following types: - - string. It will be interpreted as a Sinatra-like pattern. In - particular, the following syntax is recognized: - - a path segment starting with with a colon will match any - string placed at that position. e.g., "/:name" will match - "/carl", binding "name" to "carl". - - a pattern ending with "/*" will match any route with that - prefix. For instance, the pattern "/u/:name/*" will match - "/u/carl/" and "/u/carl/projects/123", but not "/u/carl" - (because there is no trailing slash). In addition to any names - bound in the pattern, the special name "*" is bound to the - unmatched tail of the match, but including the leading "/". So - for the two matching examples above, "*" would be bound to "/" - and "/projects/123" respectively. - - regexp.Regexp. The library assumes that it is a Perl-style regexp that - is anchored on the left (i.e., the beginning of the string). If your - regexp is not anchored on the left, a hopefully-identical - left-anchored regexp will be created and used instead. - - web.Pattern -Handler must be one of the following types: +and handler. Pattern will be passed to ParsePattern, which takes a web.Pattern, +a string, or a regular expression (more information can be found in the +ParsePattern documentation). Handler must be one of the following types: - http.Handler - web.Handler - func(w http.ResponseWriter, r *http.Request) diff --git a/web/pattern.go b/web/pattern.go index 0d27c51..99ca59e 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -31,8 +31,39 @@ type Pattern interface { Run(r *http.Request, c *C) } -func parsePattern(p interface{}) Pattern { - switch v := p.(type) { +/* +ParsePattern is used internally by Goji to parse route patterns. It is exposed +publicly to make it easier to write thin wrappers around the built-in Pattern +implementations. + +Although its parameter has type interface{}, ParsePattern only accepts arguments +of three types: + - web.Pattern, which is passed through + - string, which is interpreted as a Sinatra-like URL pattern. In + particular, the following syntax is recognized: + - a path segment starting with with a colon will match any + string placed at that position. e.g., "/:name" will match + "/carl", binding "name" to "carl". + - a pattern ending with "/*" will match any route with that + prefix. For instance, the pattern "/u/:name/*" will match + "/u/carl/" and "/u/carl/projects/123", but not "/u/carl" + (because there is no trailing slash). In addition to any names + bound in the pattern, the special key "*" is bound to the + unmatched tail of the match, but including the leading "/". So + for the two matching examples above, "*" would be bound to "/" + and "/projects/123" respectively. + - regexp.Regexp, which is assumed to be a Perl-style regular expression + that is anchored on the left (i.e., the beginning of the string). If + your regular expression is not anchored on the left, a + hopefully-identical left-anchored regular expression will be created + and used instead. + +ParsePattern fatally exits (using log.Fatalf) if it is passed a value of an +unexpected type. It is the caller's responsibility to ensure that ParsePattern +is called in a type-safe manner. +*/ +func ParsePattern(raw interface{}) Pattern { + switch v := raw.(type) { case Pattern: return v case *regexp.Regexp: @@ -40,8 +71,8 @@ func parsePattern(p interface{}) Pattern { case string: return parseStringPattern(v) default: - log.Fatalf("Unknown pattern type %v. Expected a web.Pattern, "+ - "regexp.Regexp, or a string.", p) + log.Fatalf("Unknown pattern type %T. Expected a web.Pattern, "+ + "regexp.Regexp, or a string.", v) } panic("log.Fatalf does not return") } diff --git a/web/router.go b/web/router.go index d4b88d5..277f524 100644 --- a/web/router.go +++ b/web/router.go @@ -141,7 +141,7 @@ func (rt *router) route(c *C, w http.ResponseWriter, r *http.Request) { } func (rt *router) handleUntyped(p interface{}, m method, h interface{}) { - rt.handle(parsePattern(p), m, parseHandler(h)) + rt.handle(ParsePattern(p), m, parseHandler(h)) } func (rt *router) handle(p Pattern, m method, h Handler) { From 986effffd7bc06c1c494f10a1b63c59f2383ed64 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 7 Dec 2014 17:05:16 -0800 Subject: [PATCH 162/217] Delete some unneeded code --- web/router.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/web/router.go b/web/router.go index 277f524..c291294 100644 --- a/web/router.go +++ b/web/router.go @@ -57,13 +57,11 @@ type router struct { notFound Handler machine *routeMachine } + type netHTTPWrap struct { http.Handler } -func (h netHTTPWrap) ServeHTTP(w http.ResponseWriter, r *http.Request) { - h.Handler.ServeHTTP(w, r) -} func (h netHTTPWrap) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { h.Handler.ServeHTTP(w, r) } From 565aaaa983763df9b5d4ed4587876fb1c676d163 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 11 Dec 2014 11:44:06 -0800 Subject: [PATCH 163/217] More Go versions in Travis; fix -cover --- .travis.yml | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 493b209..769961d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,26 @@ language: go -go: - - 1.2 - - 1.3 - - tip -install: - - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v - - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v - - go build -v ./... + +matrix: + include: + - go: 1.2 + install: + - go get code.google.com/p/go.tools/cmd/cover + - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v + - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v + - go: 1.3 + install: + - go get code.google.com/p/go.tools/cmd/cover + - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v + - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v + - go: 1.4 + install: + - go get golang.org/x/tools/cmd/cover + - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v + - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v + - go: tip + install: + - go get golang.org/x/tools/cmd/cover + - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v + - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v script: - - go test -v ./... + - go test -v ./... -cover From b805cb86673cf242ca7cf3546c712eebe3d9985e Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 6 Nov 2014 22:37:37 -0800 Subject: [PATCH 164/217] Test case for sync.WaitGroup race --- graceful/race_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 graceful/race_test.go diff --git a/graceful/race_test.go b/graceful/race_test.go new file mode 100644 index 0000000..cd48bae --- /dev/null +++ b/graceful/race_test.go @@ -0,0 +1,12 @@ +// +build race + +package graceful + +import "testing" + +func TestWaitGroupRace(t *testing.T) { + go func() { + go WrapConn(fakeConn{}).Close() + }() + Shutdown() +} From 8c222e182a4dfc3dda15ba84186a3869d7ffc7ef Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 4 Jan 2015 23:42:41 +0100 Subject: [PATCH 165/217] Make some sketchy tests Go 1.2 only In particular, these started failing when running tests under the race detector in Go 1.4 [0], probably due to some kind of (GC?) hijinks clearing out the sync.Pool. [0]: these tests might have also failed in 1.3, but I didn't check --- web/middleware12_test.go | 52 ++++++++++++++++++++++++++++++++++++++++ web/middleware_test.go | 44 ---------------------------------- 2 files changed, 52 insertions(+), 44 deletions(-) create mode 100644 web/middleware12_test.go diff --git a/web/middleware12_test.go b/web/middleware12_test.go new file mode 100644 index 0000000..fe5ca9b --- /dev/null +++ b/web/middleware12_test.go @@ -0,0 +1,52 @@ +// +build !go1.3 + +package web + +import "testing" + +// These tests were pretty sketchtacular to start with, but they aren't even +// guaranteed to pass with Go 1.3's sync.Pool. Let's keep them here for now; if +// they start spuriously failing later we can delete them outright. + +func TestCaching(t *testing.T) { + ch := make(chan string) + st := makeStack(ch) + cs1 := st.alloc() + cs2 := st.alloc() + if cs1 == cs2 { + t.Fatal("cs1 and cs2 are the same") + } + st.release(cs2) + cs3 := st.alloc() + if cs2 != cs3 { + t.Fatalf("Expected cs2 to equal cs3") + } + st.release(cs1) + st.release(cs3) + cs4 := st.alloc() + cs5 := st.alloc() + if cs4 != cs1 { + t.Fatal("Expected cs4 to equal cs1") + } + if cs5 != cs3 { + t.Fatal("Expected cs5 to equal cs3") + } +} + +func TestInvalidation(t *testing.T) { + ch := make(chan string) + st := makeStack(ch) + cs1 := st.alloc() + cs2 := st.alloc() + st.release(cs1) + st.invalidate() + cs3 := st.alloc() + if cs3 == cs1 { + t.Fatal("Expected cs3 to be fresh, instead got cs1") + } + st.release(cs2) + cs4 := st.alloc() + if cs4 == cs2 { + t.Fatal("Expected cs4 to be fresh, instead got cs2") + } +} diff --git a/web/middleware_test.go b/web/middleware_test.go index ef36ac9..b262b07 100644 --- a/web/middleware_test.go +++ b/web/middleware_test.go @@ -163,50 +163,6 @@ func TestAbandon(t *testing.T) { assertOrder(t, ch, "one", "router", "end") } -// This is a pretty sketchtacular test -func TestCaching(t *testing.T) { - ch := make(chan string) - st := makeStack(ch) - cs1 := st.alloc() - cs2 := st.alloc() - if cs1 == cs2 { - t.Fatal("cs1 and cs2 are the same") - } - st.release(cs2) - cs3 := st.alloc() - if cs2 != cs3 { - t.Fatalf("Expected cs2 to equal cs3") - } - st.release(cs1) - st.release(cs3) - cs4 := st.alloc() - cs5 := st.alloc() - if cs4 != cs1 { - t.Fatal("Expected cs4 to equal cs1") - } - if cs5 != cs3 { - t.Fatal("Expected cs5 to equal cs3") - } -} - -func TestInvalidation(t *testing.T) { - ch := make(chan string) - st := makeStack(ch) - cs1 := st.alloc() - cs2 := st.alloc() - st.release(cs1) - st.invalidate() - cs3 := st.alloc() - if cs3 == cs1 { - t.Fatal("Expected cs3 to be fresh, instead got cs1") - } - st.release(cs2) - cs4 := st.alloc() - if cs4 == cs2 { - t.Fatal("Expected cs4 to be fresh, instead got cs2") - } -} - func TestContext(t *testing.T) { router := func(c *C, w http.ResponseWriter, r *http.Request) { if c.Env["reqID"].(int) != 2 { From acf12155c0025c27d36f85045c9a5634765a1875 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 15 Dec 2014 01:08:05 -0800 Subject: [PATCH 166/217] Refactor package graceful This change refactors package graceful into two packages: one very well tested package that deals with graceful shutdown of arbitrary net.Listeners in the abstract, and one less-well-tested package that works with the nitty-gritty details of net/http and signal handling. This is a breaking API change for advanced users of package graceful: the WrapConn function no longer exists. This shouldn't affect most users or use cases. --- graceful/conn_set.go | 80 ------------- graceful/conn_test.go | 63 ---------- graceful/graceful.go | 43 ++++--- graceful/listener/conn.go | 147 +++++++++++++++++++++++ graceful/listener/conn_test.go | 182 ++++++++++++++++++++++++++++ graceful/listener/fake_test.go | 123 +++++++++++++++++++ graceful/listener/listener.go | 165 +++++++++++++++++++++++++ graceful/listener/listener_test.go | 143 ++++++++++++++++++++++ graceful/listener/race_test.go | 103 ++++++++++++++++ graceful/listener/shard.go | 71 +++++++++++ graceful/middleware.go | 11 +- graceful/middleware_test.go | 15 +-- graceful/net.go | 185 ----------------------------- graceful/race_test.go | 12 -- graceful/serve.go | 22 +--- graceful/serve13.go | 110 +++++++++-------- graceful/signal.go | 86 +++++++++----- 17 files changed, 1086 insertions(+), 475 deletions(-) delete mode 100644 graceful/conn_set.go delete mode 100644 graceful/conn_test.go create mode 100644 graceful/listener/conn.go create mode 100644 graceful/listener/conn_test.go create mode 100644 graceful/listener/fake_test.go create mode 100644 graceful/listener/listener.go create mode 100644 graceful/listener/listener_test.go create mode 100644 graceful/listener/race_test.go create mode 100644 graceful/listener/shard.go delete mode 100644 graceful/net.go delete mode 100644 graceful/race_test.go diff --git a/graceful/conn_set.go b/graceful/conn_set.go deleted file mode 100644 index c08099c..0000000 --- a/graceful/conn_set.go +++ /dev/null @@ -1,80 +0,0 @@ -package graceful - -import ( - "runtime" - "sync" -) - -type connShard struct { - mu sync.Mutex - // We sort of abuse this field to also act as a "please shut down" flag. - // If it's nil, you should die at your earliest opportunity. - set map[*conn]struct{} -} - -type connSet struct { - // This is an incrementing connection counter so we round-robin - // connections across shards. Use atomic when touching it. - id uint64 - shards []*connShard -} - -var idleSet connSet - -// We pretty aggressively preallocate set entries in the hopes that we never -// have to allocate memory with the lock held. This is definitely a premature -// optimization and is probably misguided, but luckily it costs us essentially -// nothing. -const prealloc = 2048 - -func init() { - // To keep the expected contention rate constant we'd have to grow this - // as numcpu**2. In practice, CPU counts don't generally grow without - // bound, and contention is probably going to be small enough that - // nobody cares anyways. - idleSet.shards = make([]*connShard, 2*runtime.NumCPU()) - for i := range idleSet.shards { - idleSet.shards[i] = &connShard{ - set: make(map[*conn]struct{}, prealloc), - } - } -} - -func (cs connSet) markIdle(c *conn) { - c.busy = false - shard := cs.shards[int(c.id%uint64(len(cs.shards)))] - shard.mu.Lock() - if shard.set == nil { - shard.mu.Unlock() - c.die = true - } else { - shard.set[c] = struct{}{} - shard.mu.Unlock() - } -} - -func (cs connSet) markBusy(c *conn) { - c.busy = true - shard := cs.shards[int(c.id%uint64(len(cs.shards)))] - shard.mu.Lock() - if shard.set == nil { - shard.mu.Unlock() - c.die = true - } else { - delete(shard.set, c) - shard.mu.Unlock() - } -} - -func (cs connSet) killall() { - for _, shard := range cs.shards { - shard.mu.Lock() - set := shard.set - shard.set = nil - shard.mu.Unlock() - - for conn := range set { - conn.closeIfIdle() - } - } -} diff --git a/graceful/conn_test.go b/graceful/conn_test.go deleted file mode 100644 index d86f12e..0000000 --- a/graceful/conn_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package graceful - -import ( - "net" - "time" -) - -// Stub out a net.Conn. This is going to be painful. - -type fakeAddr struct{} - -func (f fakeAddr) Network() string { - return "fake" -} -func (f fakeAddr) String() string { - return "fake" -} - -type fakeConn struct { - onRead, onWrite, onClose, onLocalAddr, onRemoteAddr func() - onSetDeadline, onSetReadDeadline, onSetWriteDeadline func() -} - -// Here's my number, so... -func callMeMaybe(f func()) { - // I apologize for nothing. - if f != nil { - f() - } -} - -func (f fakeConn) Read(b []byte) (int, error) { - callMeMaybe(f.onRead) - return len(b), nil -} -func (f fakeConn) Write(b []byte) (int, error) { - callMeMaybe(f.onWrite) - return len(b), nil -} -func (f fakeConn) Close() error { - callMeMaybe(f.onClose) - return nil -} -func (f fakeConn) LocalAddr() net.Addr { - callMeMaybe(f.onLocalAddr) - return fakeAddr{} -} -func (f fakeConn) RemoteAddr() net.Addr { - callMeMaybe(f.onRemoteAddr) - return fakeAddr{} -} -func (f fakeConn) SetDeadline(t time.Time) error { - callMeMaybe(f.onSetDeadline) - return nil -} -func (f fakeConn) SetReadDeadline(t time.Time) error { - callMeMaybe(f.onSetReadDeadline) - return nil -} -func (f fakeConn) SetWriteDeadline(t time.Time) error { - callMeMaybe(f.onSetWriteDeadline) - return nil -} diff --git a/graceful/graceful.go b/graceful/graceful.go index 8958126..98edd8b 100644 --- a/graceful/graceful.go +++ b/graceful/graceful.go @@ -3,18 +3,7 @@ Package graceful implements graceful shutdown for HTTP servers by closing idle connections after receiving a signal. By default, this package listens for interrupts (i.e., SIGINT), but when it detects that it is running under Einhorn it will additionally listen for SIGUSR2 as well, giving your application -automatic support for graceful upgrades. - -It's worth mentioning explicitly that this package is a hack to shim graceful -shutdown behavior into the net/http package provided in Go 1.2. It was written -by carefully reading the sequence of function calls net/http happened to use as -of this writing and finding enough surface area with which to add appropriate -behavior. There's a very good chance that this package will cease to work in -future versions of Go, but with any luck the standard library will add support -of its own by then (https://code.google.com/p/go/issues/detail?id=4674). - -If you're interested in figuring out how this package works, we suggest you read -the documentation for WrapConn() and net.go. +automatic support for graceful restarts/code upgrades. */ package graceful @@ -22,19 +11,11 @@ import ( "crypto/tls" "net" "net/http" -) -/* -You might notice that these methods look awfully similar to the methods of the -same name from the go standard library--that's because they were stolen from -there! If go were more like, say, Ruby, it'd actually be possible to shim just -the Serve() method, since we can do everything we want from there. However, it's -not possible to get the other methods which call Serve() (ListenAndServe(), say) -to call your shimmed copy--they always call the original. + "github.com/zenazn/goji/graceful/listener" +) -Since I couldn't come up with a better idea, I just copy-and-pasted both -ListenAndServe and ListenAndServeTLS here more-or-less verbatim. "Oh well!" -*/ +// Most of the code here is lifted straight from net/http // Type Server is exactly the same as an http.Server, but provides more graceful // implementations of its methods. @@ -98,3 +79,19 @@ func Serve(l net.Listener, handler http.Handler) error { server := &Server{Handler: handler} return server.Serve(l) } + +// WrapListener wraps an arbitrary net.Listener for use with graceful shutdowns. +// In the background, it uses the listener sub-package to Wrap the listener in +// Deadline mode. If another mode of operation is desired, you should call +// listener.Wrap yourself: this function is smart enough to not double-wrap +// listeners. +func WrapListener(l net.Listener) net.Listener { + if lt, ok := l.(*listener.T); ok { + appendListener(lt) + return lt + } + + lt := listener.Wrap(l, listener.Deadline) + appendListener(lt) + return lt +} diff --git a/graceful/listener/conn.go b/graceful/listener/conn.go new file mode 100644 index 0000000..22e986f --- /dev/null +++ b/graceful/listener/conn.go @@ -0,0 +1,147 @@ +package listener + +import ( + "errors" + "io" + "net" + "sync" + "time" +) + +type conn struct { + net.Conn + + shard *shard + mode mode + + mu sync.Mutex // Protects the state machine below + busy bool // connection is in use (i.e., not idle) + closed bool // connection is closed + disowned bool // if true, this connection is no longer under our management +} + +// This intentionally looks a lot like the one in package net. +var errClosing = errors.New("use of closed network connection") + +func (c *conn) init() error { + c.shard.wg.Add(1) + if shouldExit := c.shard.markIdle(c); shouldExit { + c.Close() + return errClosing + } + return nil +} + +func (c *conn) Read(b []byte) (n int, err error) { + defer func() { + c.mu.Lock() + defer c.mu.Unlock() + + if c.disowned { + return + } + + // This protects against a Close/Read race. We're not really + // concerned about the general case (it's fundamentally racy), + // but are mostly trying to prevent a race between a new request + // getting read off the wire in one thread while the connection + // is being gracefully shut down in another. + if c.closed && err == nil { + n = 0 + err = errClosing + return + } + + if c.mode != Manual && !c.busy && !c.closed { + c.busy = true + c.shard.markInUse(c) + } + }() + + return c.Conn.Read(b) +} + +func (c *conn) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.closed && !c.disowned { + c.closed = true + c.shard.markInUse(c) + defer c.shard.wg.Done() + } + + return c.Conn.Close() +} + +func (c *conn) SetReadDeadline(t time.Time) error { + c.mu.Lock() + if !c.disowned && c.mode == Deadline { + defer c.markIdle() + } + c.mu.Unlock() + return c.Conn.SetReadDeadline(t) +} + +func (c *conn) ReadFrom(r io.Reader) (int64, error) { + return io.Copy(c.Conn, r) +} + +func (c *conn) markIdle() { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.busy { + return + } + c.busy = false + + if exit := c.shard.markIdle(c); exit && !c.closed && !c.disowned { + c.closed = true + c.shard.markInUse(c) + defer c.shard.wg.Done() + c.Conn.Close() + return + } +} + +func (c *conn) markInUse() { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.busy && !c.closed && !c.disowned { + c.busy = true + c.shard.markInUse(c) + } +} + +func (c *conn) closeIfIdle() error { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.busy && !c.closed && !c.disowned { + c.closed = true + c.shard.markInUse(c) + defer c.shard.wg.Done() + return c.Conn.Close() + } + + return nil +} + +var errAlreadyDisowned = errors.New("listener: conn already disowned") + +func (c *conn) disown() error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.disowned { + return errAlreadyDisowned + } + + c.shard.markInUse(c) + c.disowned = true + c.shard.wg.Done() + + return nil +} diff --git a/graceful/listener/conn_test.go b/graceful/listener/conn_test.go new file mode 100644 index 0000000..cd5d878 --- /dev/null +++ b/graceful/listener/conn_test.go @@ -0,0 +1,182 @@ +package listener + +import ( + "io" + "strings" + "testing" + "time" +) + +func TestManualRead(t *testing.T) { + t.Parallel() + l, c, wc := singleConn(t, Manual) + + go c.AllowRead() + wc.Read(make([]byte, 1024)) + + if err := l.CloseIdle(); err != nil { + t.Fatalf("error closing idle connections: %v", err) + } + if !c.Closed() { + t.Error("Read() should not make connection not-idle") + } +} + +func TestAutomaticRead(t *testing.T) { + t.Parallel() + l, c, wc := singleConn(t, Automatic) + + go c.AllowRead() + wc.Read(make([]byte, 1024)) + + if err := l.CloseIdle(); err != nil { + t.Fatalf("error closing idle connections: %v", err) + } + if c.Closed() { + t.Error("expected Read() to mark connection as in-use") + } +} + +func TestDeadlineRead(t *testing.T) { + t.Parallel() + l, c, wc := singleConn(t, Deadline) + + go c.AllowRead() + if _, err := wc.Read(make([]byte, 1024)); err != nil { + t.Fatalf("error reading from connection: %v", err) + } + + if err := l.CloseIdle(); err != nil { + t.Fatalf("error closing idle connections: %v", err) + } + if c.Closed() { + t.Error("expected Read() to mark connection as in-use") + } +} + +func TestDisownedRead(t *testing.T) { + t.Parallel() + l, c, wc := singleConn(t, Deadline) + + if err := Disown(wc); err != nil { + t.Fatalf("unexpected error disowning conn: %v", err) + } + if err := l.Close(); err != nil { + t.Fatalf("unexpected error closing listener: %v", err) + } + if err := l.Drain(); err != nil { + t.Fatalf("unexpected error draining listener: %v", err) + } + + go c.AllowRead() + if _, err := wc.Read(make([]byte, 1024)); err != nil { + t.Fatalf("error reading from connection: %v", err) + } +} + +func TestCloseConn(t *testing.T) { + t.Parallel() + l, _, wc := singleConn(t, Deadline) + + if err := MarkInUse(wc); err != nil { + t.Fatalf("error marking conn in use: %v", err) + } + if err := wc.Close(); err != nil { + t.Errorf("error closing connection: %v", err) + } + // This will hang if wc.Close() doesn't un-track the connection + if err := l.Drain(); err != nil { + t.Errorf("error draining listener: %v", err) + } +} + +func TestManualReadDeadline(t *testing.T) { + t.Parallel() + l, c, wc := singleConn(t, Manual) + + if err := MarkInUse(wc); err != nil { + t.Fatalf("error marking connection in use: %v", err) + } + if err := wc.SetReadDeadline(time.Now()); err != nil { + t.Fatalf("error setting read deadline: %v", err) + } + if err := l.CloseIdle(); err != nil { + t.Fatalf("error closing idle connections: %v", err) + } + if c.Closed() { + t.Error("SetReadDeadline() should not mark manual conn as idle") + } +} + +func TestAutomaticReadDeadline(t *testing.T) { + t.Parallel() + l, c, wc := singleConn(t, Automatic) + + if err := MarkInUse(wc); err != nil { + t.Fatalf("error marking connection in use: %v", err) + } + if err := wc.SetReadDeadline(time.Now()); err != nil { + t.Fatalf("error setting read deadline: %v", err) + } + if err := l.CloseIdle(); err != nil { + t.Fatalf("error closing idle connections: %v", err) + } + if c.Closed() { + t.Error("SetReadDeadline() should not mark automatic conn as idle") + } +} + +func TestDeadlineReadDeadline(t *testing.T) { + t.Parallel() + l, c, wc := singleConn(t, Deadline) + + if err := MarkInUse(wc); err != nil { + t.Fatalf("error marking connection in use: %v", err) + } + if err := wc.SetReadDeadline(time.Now()); err != nil { + t.Fatalf("error setting read deadline: %v", err) + } + if err := l.CloseIdle(); err != nil { + t.Fatalf("error closing idle connections: %v", err) + } + if !c.Closed() { + t.Error("SetReadDeadline() should mark deadline conn as idle") + } +} + +type readerConn struct { + fakeConn +} + +func (rc *readerConn) ReadFrom(r io.Reader) (int64, error) { + return 123, nil +} + +func TestReadFrom(t *testing.T) { + t.Parallel() + + l := makeFakeListener("net.Listener") + wl := Wrap(l, Manual) + c := &readerConn{ + fakeConn{ + read: make(chan struct{}), + write: make(chan struct{}), + closed: make(chan struct{}), + me: fakeAddr{"tcp", "local"}, + you: fakeAddr{"tcp", "remote"}, + }, + } + + go l.Enqueue(c) + wc, err := wl.Accept() + if err != nil { + t.Fatalf("error accepting connection: %v", err) + } + + // The io.MultiReader is a convenient hack to ensure that we're using + // our ReadFrom, not strings.Reader's WriteTo. + r := io.MultiReader(strings.NewReader("hello world")) + if _, err := io.Copy(wc, r); err != nil { + t.Fatalf("error copying: %v", err) + } +} diff --git a/graceful/listener/fake_test.go b/graceful/listener/fake_test.go new file mode 100644 index 0000000..083f6a8 --- /dev/null +++ b/graceful/listener/fake_test.go @@ -0,0 +1,123 @@ +package listener + +import ( + "net" + "time" +) + +type fakeAddr struct { + network, addr string +} + +func (f fakeAddr) Network() string { + return f.network +} +func (f fakeAddr) String() string { + return f.addr +} + +type fakeListener struct { + ch chan net.Conn + closed chan struct{} + addr net.Addr +} + +func makeFakeListener(addr string) *fakeListener { + a := fakeAddr{"tcp", addr} + return &fakeListener{ + ch: make(chan net.Conn), + closed: make(chan struct{}), + addr: a, + } +} + +func (f *fakeListener) Accept() (net.Conn, error) { + select { + case c := <-f.ch: + return c, nil + case <-f.closed: + return nil, errClosing + } +} +func (f *fakeListener) Close() error { + close(f.closed) + return nil +} + +func (f *fakeListener) Addr() net.Addr { + return f.addr +} + +func (f *fakeListener) Enqueue(c net.Conn) { + f.ch <- c +} + +type fakeConn struct { + read, write, closed chan struct{} + me, you net.Addr +} + +func makeFakeConn(me, you string) *fakeConn { + return &fakeConn{ + read: make(chan struct{}), + write: make(chan struct{}), + closed: make(chan struct{}), + me: fakeAddr{"tcp", me}, + you: fakeAddr{"tcp", you}, + } +} + +func (f *fakeConn) Read(buf []byte) (int, error) { + select { + case <-f.read: + return len(buf), nil + case <-f.closed: + return 0, errClosing + } +} + +func (f *fakeConn) Write(buf []byte) (int, error) { + select { + case <-f.write: + return len(buf), nil + case <-f.closed: + return 0, errClosing + } +} + +func (f *fakeConn) Close() error { + close(f.closed) + return nil +} + +func (f *fakeConn) LocalAddr() net.Addr { + return f.me +} +func (f *fakeConn) RemoteAddr() net.Addr { + return f.you +} +func (f *fakeConn) SetDeadline(t time.Time) error { + return nil +} +func (f *fakeConn) SetReadDeadline(t time.Time) error { + return nil +} +func (f *fakeConn) SetWriteDeadline(t time.Time) error { + return nil +} + +func (f *fakeConn) Closed() bool { + select { + case <-f.closed: + return true + default: + return false + } +} + +func (f *fakeConn) AllowRead() { + f.read <- struct{}{} +} +func (f *fakeConn) AllowWrite() { + f.write <- struct{}{} +} diff --git a/graceful/listener/listener.go b/graceful/listener/listener.go new file mode 100644 index 0000000..bebfb31 --- /dev/null +++ b/graceful/listener/listener.go @@ -0,0 +1,165 @@ +/* +Package listener provides a way to incorporate graceful shutdown to any +net.Listener. + +This package provides low-level primitives, not a high-level API. If you're +looking for a package that provides graceful shutdown for HTTP servers, I +recommend this package's parent package, github.com/zenazn/goji/graceful. +*/ +package listener + +import ( + "errors" + "net" + "runtime" + "sync" + "sync/atomic" +) + +type mode int8 + +const ( + // Manual mode is completely manual: users must use use MarkIdle and + // MarkInUse to indicate when connections are busy servicing requests or + // are eligible for termination. + Manual mode = iota + // Automatic mode is what most users probably want: calling Read on a + // connection will mark it as in use, but users must manually call + // MarkIdle to indicate when connections may be safely closed. + Automatic + // Deadline mode is like automatic mode, except that calling + // SetReadDeadline on a connection will also mark it as being idle. This + // is useful for many servers like net/http, where SetReadDeadline is + // used to implement read timeouts on new requests. + Deadline +) + +// Wrap a net.Listener, returning a net.Listener which supports idle connection +// tracking and shutdown. Listeners can be placed in to one of three modes, +// exported as variables from this package: most users will probably want the +// "Automatic" mode. +func Wrap(l net.Listener, m mode) *T { + t := &T{ + l: l, + mode: m, + // To keep the expected contention rate constant we'd have to + // grow this as numcpu**2. In practice, CPU counts don't + // generally grow without bound, and contention is probably + // going to be small enough that nobody cares anyways. + shards: make([]shard, 2*runtime.NumCPU()), + } + for i := range t.shards { + t.shards[i].init(t) + } + return t +} + +// T is the type of this package's graceful listeners. +type T struct { + mu sync.Mutex + l net.Listener + + // TODO(carl): a count of currently outstanding connections. + connCount uint64 + shards []shard + + mode mode +} + +var _ net.Listener = &T{} + +// Accept waits for and returns the next connection to the listener. The +// returned net.Conn's idleness is tracked, and idle connections can be closed +// from the associated T. +func (t *T) Accept() (net.Conn, error) { + c, err := t.l.Accept() + if err != nil { + return nil, err + } + + connID := atomic.AddUint64(&t.connCount, 1) + shard := &t.shards[int(connID)%len(t.shards)] + wc := &conn{ + Conn: c, + shard: shard, + mode: t.mode, + } + + if err = wc.init(); err != nil { + return nil, err + } + return wc, nil +} + +// Addr returns the wrapped listener's network address. +func (t *T) Addr() net.Addr { + return t.l.Addr() +} + +// Close closes the wrapped listener. +func (t *T) Close() error { + return t.l.Close() +} + +// CloseIdle closes all connections that are currently marked as being idle. It, +// however, makes no attempt to wait for in-use connections to die, or to close +// connections which become idle in the future. Call this function if you're +// interested in shedding useless connections, but otherwise wish to continue +// serving requests. +func (t *T) CloseIdle() error { + for i := range t.shards { + t.shards[i].closeIdle(false) + } + // Not sure if returning errors is actually useful here :/ + return nil +} + +// Drain immediately closes all idle connections, prevents new connections from +// being accepted, and waits for all outstanding connections to finish. +// +// Once a listener has been drained, there is no way to re-enable it. You +// probably want to Close the listener before draining it, otherwise new +// connections will be accepted and immediately closed. +func (t *T) Drain() error { + for i := range t.shards { + t.shards[i].closeIdle(true) + } + for i := range t.shards { + t.shards[i].wait() + } + return nil +} + +var notManagedErr = errors.New("listener: passed net.Conn is not managed by us") + +// Disown causes a connection to no longer be tracked by the listener. The +// passed connection must have been returned by a call to Accept from this +// listener. +func Disown(c net.Conn) error { + if cn, ok := c.(*conn); ok { + return cn.disown() + } + return notManagedErr +} + +// MarkIdle marks the given connection as being idle, and therefore eligible for +// closing at any time. The passed connection must have been returned by a call +// to Accept from this listener. +func MarkIdle(c net.Conn) error { + if cn, ok := c.(*conn); ok { + cn.markIdle() + return nil + } + return notManagedErr +} + +// MarkInUse marks this connection as being in use, removing it from the set of +// connections which are eligible for closing. The passed connection must have +// been returned by a call to Accept from this listener. +func MarkInUse(c net.Conn) error { + if cn, ok := c.(*conn); ok { + cn.markInUse() + return nil + } + return notManagedErr +} diff --git a/graceful/listener/listener_test.go b/graceful/listener/listener_test.go new file mode 100644 index 0000000..160434b --- /dev/null +++ b/graceful/listener/listener_test.go @@ -0,0 +1,143 @@ +package listener + +import ( + "net" + "testing" + "time" +) + +// Helper for tests acting on a single accepted connection +func singleConn(t *testing.T, m mode) (*T, *fakeConn, net.Conn) { + l := makeFakeListener("net.Listener") + wl := Wrap(l, m) + c := makeFakeConn("local", "remote") + + go l.Enqueue(c) + wc, err := wl.Accept() + if err != nil { + t.Fatalf("error accepting connection: %v", err) + } + return wl, c, wc +} + +func TestAddr(t *testing.T) { + t.Parallel() + l, c, wc := singleConn(t, Manual) + + if a := l.Addr(); a.String() != "net.Listener" { + t.Errorf("addr was %v, wanted net.Listener", a) + } + + if c.LocalAddr() != wc.LocalAddr() { + t.Errorf("local addresses don't match: %v, %v", c.LocalAddr(), + wc.LocalAddr()) + } + if c.RemoteAddr() != wc.RemoteAddr() { + t.Errorf("remote addresses don't match: %v, %v", c.RemoteAddr(), + wc.RemoteAddr()) + } +} + +func TestBasicCloseIdle(t *testing.T) { + t.Parallel() + l, c, _ := singleConn(t, Manual) + + if err := l.CloseIdle(); err != nil { + t.Fatalf("error closing idle connections: %v", err) + } + if !c.Closed() { + t.Error("idle connection not closed") + } +} + +func TestMark(t *testing.T) { + t.Parallel() + l, c, wc := singleConn(t, Manual) + + if err := MarkInUse(wc); err != nil { + t.Fatalf("error marking %v in-use: %v", wc, err) + } + if err := l.CloseIdle(); err != nil { + t.Fatalf("error closing idle connections: %v", err) + } + if c.Closed() { + t.Errorf("manually in-use connection was closed") + } + + if err := MarkIdle(wc); err != nil { + t.Fatalf("error marking %v idle: %v", wc, err) + } + if err := l.CloseIdle(); err != nil { + t.Fatalf("error closing idle connections: %v", err) + } + if !c.Closed() { + t.Error("manually idle connection was not closed") + } +} + +func TestDisown(t *testing.T) { + t.Parallel() + l, c, wc := singleConn(t, Manual) + + if err := Disown(wc); err != nil { + t.Fatalf("error disowning connection: %v", err) + } + if err := l.CloseIdle(); err != nil { + t.Fatalf("error closing idle connections: %v", err) + } + + if c.Closed() { + t.Errorf("disowned connection got closed") + } +} + +func TestDrain(t *testing.T) { + t.Parallel() + l, _, wc := singleConn(t, Manual) + + MarkInUse(wc) + start := time.Now() + go func() { + time.Sleep(50 * time.Millisecond) + MarkIdle(wc) + }() + if err := l.Drain(); err != nil { + t.Fatalf("error draining listener: %v", err) + } + end := time.Now() + if dt := end.Sub(start); dt < 50*time.Millisecond { + t.Errorf("expected at least 50ms wait, but got %v", dt) + } +} + +func TestErrors(t *testing.T) { + t.Parallel() + _, c, wc := singleConn(t, Manual) + if err := Disown(c); err == nil { + t.Error("expected error when disowning unmanaged net.Conn") + } + if err := MarkIdle(c); err == nil { + t.Error("expected error when marking unmanaged net.Conn idle") + } + if err := MarkInUse(c); err == nil { + t.Error("expected error when marking unmanaged net.Conn in use") + } + + if err := Disown(wc); err != nil { + t.Fatalf("unexpected error disowning socket: %v", err) + } + if err := Disown(wc); err == nil { + t.Error("expected error disowning socket twice") + } +} + +func TestClose(t *testing.T) { + t.Parallel() + l, c, _ := singleConn(t, Manual) + if err := l.Close(); err != nil { + t.Fatalf("error while closing listener: %v", err) + } + if c.Closed() { + t.Error("connection closed when listener was?") + } +} diff --git a/graceful/listener/race_test.go b/graceful/listener/race_test.go new file mode 100644 index 0000000..835d6b4 --- /dev/null +++ b/graceful/listener/race_test.go @@ -0,0 +1,103 @@ +package listener + +import ( + "fmt" + "math/rand" + "runtime" + "sync/atomic" + "testing" + "time" +) + +func init() { + // Just to make sure we get some variety + runtime.GOMAXPROCS(4 * runtime.NumCPU()) +} + +// Chosen by random die roll +const seed = 4611413766552969250 + +// This is mostly just fuzzing to see what happens. +func TestRace(t *testing.T) { + t.Parallel() + + l := makeFakeListener("net.Listener") + wl := Wrap(l, Automatic) + + var flag int32 + + go func() { + for i := 0; ; i++ { + laddr := fmt.Sprintf("local%d", i) + raddr := fmt.Sprintf("remote%d", i) + c := makeFakeConn(laddr, raddr) + go func() { + defer func() { + if r := recover(); r != nil { + if atomic.LoadInt32(&flag) != 0 { + return + } + panic(r) + } + }() + l.Enqueue(c) + }() + wc, err := wl.Accept() + if err != nil { + if atomic.LoadInt32(&flag) != 0 { + return + } + t.Fatalf("error accepting connection: %v", err) + } + + go func() { + for { + time.Sleep(50 * time.Millisecond) + c.AllowRead() + } + }() + + go func(i int64) { + rng := rand.New(rand.NewSource(i + seed)) + buf := make([]byte, 1024) + for j := 0; j < 1024; j++ { + if _, err := wc.Read(buf); err != nil { + if atomic.LoadInt32(&flag) != 0 { + // Peaceful; the connection has + // probably been closed while + // idle + return + } + t.Errorf("error reading in conn %d: %v", + i, err) + } + time.Sleep(time.Duration(rng.Intn(100)) * time.Millisecond) + // This one is to make sure the connection + // hasn't closed underneath us + if _, err := wc.Read(buf); err != nil { + t.Errorf("error reading in conn %d: %v", + i, err) + } + MarkIdle(wc) + time.Sleep(time.Duration(rng.Intn(100)) * time.Millisecond) + } + }(int64(i)) + + time.Sleep(time.Duration(i) * time.Millisecond / 2) + } + }() + + if testing.Short() { + time.Sleep(2 * time.Second) + } else { + time.Sleep(10 * time.Second) + } + start := time.Now() + atomic.StoreInt32(&flag, 1) + wl.Close() + wl.Drain() + end := time.Now() + if dt := end.Sub(start); dt > 300*time.Millisecond { + t.Errorf("took %v to drain; expected shorter", dt) + } +} diff --git a/graceful/listener/shard.go b/graceful/listener/shard.go new file mode 100644 index 0000000..e47deac --- /dev/null +++ b/graceful/listener/shard.go @@ -0,0 +1,71 @@ +package listener + +import "sync" + +type shard struct { + l *T + + mu sync.Mutex + set map[*conn]struct{} + wg sync.WaitGroup + drain bool + + // We pack shards together in an array, but we don't want them packed + // too closely, since we want to give each shard a dedicated CPU cache + // line. This amount of padding works out well for the common case of + // x64 processors (64-bit pointers with a 64-byte cache line). + _ [12]byte +} + +// We pretty aggressively preallocate set entries in the hopes that we never +// have to allocate memory with the lock held. This is definitely a premature +// optimization and is probably misguided, but luckily it costs us essentially +// nothing. +const prealloc = 2048 + +func (s *shard) init(l *T) { + s.l = l + s.set = make(map[*conn]struct{}, prealloc) +} + +func (s *shard) markIdle(c *conn) (shouldClose bool) { + s.mu.Lock() + if s.drain { + s.mu.Unlock() + return true + } + s.set[c] = struct{}{} + s.mu.Unlock() + return false +} + +func (s *shard) markInUse(c *conn) { + s.mu.Lock() + delete(s.set, c) + s.mu.Unlock() +} + +func (s *shard) closeIdle(drain bool) { + s.mu.Lock() + if drain { + s.drain = true + } + set := s.set + s.set = make(map[*conn]struct{}, prealloc) + // We have to drop the shard lock here to avoid deadlock: we cannot + // acquire the shard lock after the connection lock, and the closeIfIdle + // call below will grab a connection lock. + s.mu.Unlock() + + for conn := range set { + // This might return an error (from Close), but I don't think we + // can do anything about it, so let's just pretend it didn't + // happen. (I also expect that most errors returned in this way + // are going to be pretty boring) + conn.closeIfIdle() + } +} + +func (s *shard) wait() { + s.wg.Wait() +} diff --git a/graceful/middleware.go b/graceful/middleware.go index 4aace7d..7f1371e 100644 --- a/graceful/middleware.go +++ b/graceful/middleware.go @@ -7,6 +7,9 @@ import ( "io" "net" "net/http" + "sync/atomic" + + "github.com/zenazn/goji/graceful/listener" ) /* @@ -62,10 +65,8 @@ type basicWriter struct { func (b *basicWriter) maybeClose() { b.headerWritten = true - select { - case <-kill: + if atomic.LoadInt32(&closing) != 0 { b.ResponseWriter.Header().Set("Connection", "close") - default: } } @@ -103,8 +104,8 @@ func (f *fancyWriter) Hijack() (c net.Conn, b *bufio.ReadWriter, e error) { hj := f.basicWriter.ResponseWriter.(http.Hijacker) c, b, e = hj.Hijack() - if conn, ok := c.(*conn); ok { - c = conn.hijack() + if e == nil { + e = listener.Disown(c) } return diff --git a/graceful/middleware_test.go b/graceful/middleware_test.go index 15b4fcc..fe336f4 100644 --- a/graceful/middleware_test.go +++ b/graceful/middleware_test.go @@ -4,6 +4,7 @@ package graceful import ( "net/http" + "sync/atomic" "testing" ) @@ -36,7 +37,7 @@ func testClose(t *testing.T, h http.Handler, expectClose bool) { } func TestNormal(t *testing.T) { - kill = make(chan struct{}) + atomic.StoreInt32(&closing, 0) h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte{}) }) @@ -44,26 +45,26 @@ func TestNormal(t *testing.T) { } func TestClose(t *testing.T) { - kill = make(chan struct{}) + atomic.StoreInt32(&closing, 0) h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - close(kill) + atomic.StoreInt32(&closing, 1) }) testClose(t, h, true) } func TestCloseWriteHeader(t *testing.T) { - kill = make(chan struct{}) + atomic.StoreInt32(&closing, 0) h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - close(kill) + atomic.StoreInt32(&closing, 1) w.WriteHeader(200) }) testClose(t, h, true) } func TestCloseWrite(t *testing.T) { - kill = make(chan struct{}) + atomic.StoreInt32(&closing, 0) h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - close(kill) + atomic.StoreInt32(&closing, 1) w.Write([]byte{}) }) testClose(t, h, true) diff --git a/graceful/net.go b/graceful/net.go deleted file mode 100644 index 784003d..0000000 --- a/graceful/net.go +++ /dev/null @@ -1,185 +0,0 @@ -package graceful - -import ( - "io" - "net" - "sync" - "sync/atomic" - "time" -) - -type listener struct { - net.Listener -} - -// WrapListener wraps an arbitrary net.Listener for use with graceful shutdowns. -// All net.Conn's Accept()ed by this listener will be auto-wrapped as if -// WrapConn() were called on them. -func WrapListener(l net.Listener) net.Listener { - return listener{l} -} - -func (l listener) Accept() (net.Conn, error) { - conn, err := l.Listener.Accept() - return WrapConn(conn), err -} - -type conn struct { - mu sync.Mutex - cs *connSet - net.Conn - id uint64 - busy, die bool - dead bool - hijacked bool -} - -/* -WrapConn wraps an arbitrary connection for use with graceful shutdowns. The -graceful shutdown process will ensure that this connection is closed before -terminating the process. - -In order to use this function, you must call SetReadDeadline() before the call -to Read() you might make to read a new request off the wire. The connection is -eligible for abrupt closing at any point between when the call to -SetReadDeadline() returns and when the call to Read returns with new data. It -does not matter what deadline is given to SetReadDeadline()--if a deadline is -inappropriate, providing one extremely far into the future will suffice. - -Unfortunately, this means that it's difficult to use SetReadDeadline() in a -great many perfectly reasonable circumstances, such as to extend a deadline -after more data has been read, without the connection being eligible for -"graceful" termination at an undesirable time. Since this package was written -explicitly to target net/http, which does not as of this writing do any of this, -fixing the semantics here does not seem especially urgent. -*/ -func WrapConn(c net.Conn) net.Conn { - if c == nil { - return nil - } - - // Avoid race with termination code. - wgLock.Lock() - defer wgLock.Unlock() - - // Determine whether the app is shutting down. - if acceptingRequests { - wg.Add(1) - return &conn{ - Conn: c, - id: atomic.AddUint64(&idleSet.id, 1), - } - } else { - return nil - } -} - -func (c *conn) Read(b []byte) (n int, err error) { - c.mu.Lock() - if !c.hijacked { - defer func() { - c.mu.Lock() - if c.hijacked { - // It's a little unclear to me how this case - // would happen, but we *did* drop the lock, so - // let's play it safe. - return - } - - if c.dead { - // Dead sockets don't tell tales. This is to - // prevent the case where a Read manages to suck - // an entire request off the wire in a race with - // someone trying to close idle connections. - // Whoever grabs the conn lock first wins, and - // if that's the closing process, we need to - // "take back" the read. - n = 0 - err = io.EOF - } else { - idleSet.markBusy(c) - } - c.mu.Unlock() - }() - } - c.mu.Unlock() - - return c.Conn.Read(b) -} - -func (c *conn) SetReadDeadline(t time.Time) error { - c.mu.Lock() - if !c.hijacked { - defer c.markIdle() - } - c.mu.Unlock() - return c.Conn.SetReadDeadline(t) -} - -func (c *conn) Close() error { - kill := false - c.mu.Lock() - kill, c.dead = !c.dead, true - idleSet.markBusy(c) - c.mu.Unlock() - - if kill { - defer wg.Done() - } - return c.Conn.Close() -} - -type writerOnly struct { - w io.Writer -} - -func (w writerOnly) Write(buf []byte) (int, error) { - return w.w.Write(buf) -} - -func (c *conn) ReadFrom(r io.Reader) (int64, error) { - if rf, ok := c.Conn.(io.ReaderFrom); ok { - return rf.ReadFrom(r) - } - return io.Copy(writerOnly{c}, r) -} - -func (c *conn) markIdle() { - kill := false - c.mu.Lock() - idleSet.markIdle(c) - if c.die { - kill, c.dead = !c.dead, true - } - c.mu.Unlock() - - if kill { - defer wg.Done() - c.Conn.Close() - } - -} - -func (c *conn) closeIfIdle() { - kill := false - c.mu.Lock() - c.die = true - if !c.busy && !c.hijacked { - kill, c.dead = !c.dead, true - } - c.mu.Unlock() - - if kill { - defer wg.Done() - c.Conn.Close() - } -} - -func (c *conn) hijack() net.Conn { - c.mu.Lock() - idleSet.markBusy(c) - c.hijacked = true - c.mu.Unlock() - - return c.Conn -} diff --git a/graceful/race_test.go b/graceful/race_test.go deleted file mode 100644 index cd48bae..0000000 --- a/graceful/race_test.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build race - -package graceful - -import "testing" - -func TestWaitGroupRace(t *testing.T) { - go func() { - go WrapConn(fakeConn{}).Close() - }() - Shutdown() -} diff --git a/graceful/serve.go b/graceful/serve.go index 626f807..f4ed19f 100644 --- a/graceful/serve.go +++ b/graceful/serve.go @@ -6,19 +6,14 @@ import ( "net" "net/http" "time" + + "github.com/zenazn/goji/graceful/listener" ) // About 200 years, also known as "forever" const forever time.Duration = 200 * 365 * 24 * time.Hour func (srv *Server) Serve(l net.Listener) error { - go func() { - <-kill - l.Close() - idleSet.killall() - }() - l = WrapListener(l) - // Spawn a shadow http.Server to do the actual servering. We do this // because we need to sketch on some of the parameters you passed in, // and it's nice to keep our sketching to ourselves. @@ -29,14 +24,9 @@ func (srv *Server) Serve(l net.Listener) error { } shadow.Handler = Middleware(shadow.Handler) - err := shadow.Serve(l) + wrap := listener.Wrap(l, listener.Deadline) + appendListener(wrap) - // We expect an error when we close the listener, so we indiscriminately - // swallow Serve errors when we're in a shutdown state. - select { - case <-kill: - return nil - default: - return err - } + err := shadow.Serve(wrap) + return peacefulError(err) } diff --git a/graceful/serve13.go b/graceful/serve13.go index e0acd18..5f5e4ed 100644 --- a/graceful/serve13.go +++ b/graceful/serve13.go @@ -3,67 +3,75 @@ package graceful import ( + "log" "net" "net/http" + + "github.com/zenazn/goji/graceful/listener" ) -func (srv *Server) Serve(l net.Listener) error { - l = WrapListener(l) +// This is a slightly hacky shim to disable keepalives when shutting a server +// down. We could have added extra functionality in listener or signal.go to +// deal with this case, but this seems simpler. +type gracefulServer struct { + net.Listener + s *http.Server +} - // Spawn a shadow http.Server to do the actual servering. We do this - // because we need to sketch on some of the parameters you passed in, - // and it's nice to keep our sketching to ourselves. - shadow := *(*http.Server)(srv) +func (g gracefulServer) Close() error { + g.s.SetKeepAlivesEnabled(false) + return g.Listener.Close() +} + +// A chaining http.ConnState wrapper +type connState func(net.Conn, http.ConnState) - cs := shadow.ConnState - shadow.ConnState = func(nc net.Conn, s http.ConnState) { - if c, ok := nc.(*conn); ok { - // There are a few other states defined, most notably - // StateActive. Unfortunately it doesn't look like it's - // possible to make use of StateActive to implement - // graceful shutdown, since StateActive is set after a - // complete request has been read off the wire with an - // intent to process it. If we were to race a graceful - // shutdown against a connection that was just read off - // the wire (but not yet in StateActive), we would - // accidentally close the connection out from underneath - // an active request. - // - // We already needed to work around this for Go 1.2 by - // shimming out a full net.Conn object, so we can just - // fall back to the old behavior there. - // - // I started a golang-nuts thread about this here: - // https://groups.google.com/forum/#!topic/golang-nuts/Xi8yjBGWfCQ - // I'd be very eager to find a better way to do this, so - // reach out to me if you have any ideas. - switch s { - case http.StateIdle: - c.markIdle() - case http.StateHijacked: - c.hijack() - } +func (c connState) Wrap(nc net.Conn, s http.ConnState) { + // There are a few other states defined, most notably StateActive. + // Unfortunately it doesn't look like it's possible to make use of + // StateActive to implement graceful shutdown, since StateActive is set + // after a complete request has been read off the wire with an intent to + // process it. If we were to race a graceful shutdown against a + // connection that was just read off the wire (but not yet in + // StateActive), we would accidentally close the connection out from + // underneath an active request. + // + // We already needed to work around this for Go 1.2 by shimming out a + // full net.Conn object, so we can just fall back to the old behavior + // there. + // + // I started a golang-nuts thread about this here: + // https://groups.google.com/forum/#!topic/golang-nuts/Xi8yjBGWfCQ I'd + // be very eager to find a better way to do this, so reach out to me if + // you have any ideas. + switch s { + case http.StateIdle: + if err := listener.MarkIdle(nc); err != nil { + log.Printf("error marking conn as idle: %v", + err) } - if cs != nil { - cs(nc, s) + case http.StateHijacked: + if err := listener.Disown(nc); err != nil { + log.Printf("error disowning hijacked conn: %v", + err) } } + if c != nil { + c(nc, s) + } +} - go func() { - <-kill - l.Close() - shadow.SetKeepAlivesEnabled(false) - idleSet.killall() - }() +func (srv *Server) Serve(l net.Listener) error { + // Spawn a shadow http.Server to do the actual servering. We do this + // because we need to sketch on some of the parameters you passed in, + // and it's nice to keep our sketching to ourselves. + shadow := *(*http.Server)(srv) + shadow.ConnState = connState(shadow.ConnState).Wrap - err := shadow.Serve(l) + l = gracefulServer{l, &shadow} + wrap := listener.Wrap(l, listener.Automatic) + appendListener(wrap) - // We expect an error when we close the listener, so we indiscriminately - // swallow Serve errors when we're in a shutdown state. - select { - case <-kill: - return nil - default: - return err - } + err := shadow.Serve(wrap) + return peacefulError(err) } diff --git a/graceful/signal.go b/graceful/signal.go index 82a13d6..6bd3d84 100644 --- a/graceful/signal.go +++ b/graceful/signal.go @@ -1,32 +1,22 @@ package graceful import ( + "net" "os" "os/signal" "sync" -) - -// This is the channel that the connections select on. When it is closed, the -// connections should gracefully exit. -var kill = make(chan struct{}) - -// This is the channel that the Wait() function selects on. It should only be -// closed once all the posthooks have been called. -var wait = make(chan struct{}) - -// Whether new requests should be accepted. When false, new requests are refused. -var acceptingRequests bool = true + "sync/atomic" -// This is the WaitGroup that indicates when all the connections have gracefully -// shut down. -var wg sync.WaitGroup -var wgLock sync.Mutex + "github.com/zenazn/goji/graceful/listener" +) -// This lock protects the list of pre- and post- hooks below. -var hookLock sync.Mutex +var mu sync.Mutex // protects everything that follows +var listeners = make([]*listener.T, 0) var prehooks = make([]func(), 0) var posthooks = make([]func(), 0) +var closing int32 +var wait = make(chan struct{}) var stdSignals = []os.Signal{os.Interrupt} var sigchan = make(chan os.Signal, 1) @@ -71,8 +61,8 @@ func Shutdown() { // shutdown actions. All listeners will be called in the order they were added, // from a single goroutine. func PreHook(f func()) { - hookLock.Lock() - defer hookLock.Unlock() + mu.Lock() + defer mu.Unlock() prehooks = append(prehooks, f) } @@ -82,12 +72,12 @@ func PreHook(f func()) { // from a single goroutine, and are guaranteed to be called after all listening // connections have been closed, but before Wait() returns. // -// If you've Hijack()ed any connections that must be gracefully shut down in -// some other way (since this library disowns all hijacked connections), it's -// reasonable to use a PostHook() to signal and wait for them. +// If you've Hijacked any connections that must be gracefully shut down in some +// other way (since this library disowns all hijacked connections), it's +// reasonable to use a PostHook to signal and wait for them. func PostHook(f func()) { - hookLock.Lock() - defer hookLock.Unlock() + mu.Lock() + defer mu.Unlock() posthooks = append(posthooks, f) } @@ -95,19 +85,23 @@ func PostHook(f func()) { func waitForSignal() { <-sigchan - // Prevent servicing of any new requests. - wgLock.Lock() - acceptingRequests = false - wgLock.Unlock() - - hookLock.Lock() - defer hookLock.Unlock() + mu.Lock() + defer mu.Unlock() for _, f := range prehooks { f() } - close(kill) + atomic.StoreInt32(&closing, 1) + var wg sync.WaitGroup + wg.Add(len(listeners)) + for _, l := range listeners { + go func(l *listener.T) { + defer wg.Done() + l.Close() + l.Drain() + }(l) + } wg.Wait() for _, f := range posthooks { @@ -123,3 +117,29 @@ func waitForSignal() { func Wait() { <-wait } + +func appendListener(l *listener.T) { + mu.Lock() + defer mu.Unlock() + + listeners = append(listeners, l) +} + +const errClosing = "use of closed network connection" + +// During graceful shutdown, calls to Accept will start returning errors. This +// is inconvenient, since we know these sorts of errors are peaceful, so we +// silently swallow them. +func peacefulError(err error) error { + if atomic.LoadInt32(&closing) == 0 { + return err + } + // Unfortunately Go doesn't really give us a better way to select errors + // than this, so *shrug*. + if oe, ok := err.(*net.OpError); ok { + if oe.Op == "accept" && oe.Err.Error() == errClosing { + return nil + } + } + return err +} From e3ea1b5780a7f3ca90b91ef05eaf04fa477c5f6f Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 5 Jan 2015 18:23:49 +0100 Subject: [PATCH 167/217] Disable SSLv3 by default This change addresses the POODLE vulnerability [0]. Unfortunately it makes package graceful's behavior here slightly different than the stock net/http methods of the same name, but I think that's fine in this situation. [0]: https://www.openssl.org/~bodo/ssl-poodle.pdf Thanks to @ekanna for reporting this. Fixes #101. --- graceful/graceful.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/graceful/graceful.go b/graceful/graceful.go index 98edd8b..255f24c 100644 --- a/graceful/graceful.go +++ b/graceful/graceful.go @@ -33,12 +33,18 @@ func (srv *Server) ListenAndServe() error { return srv.Serve(l) } +// Unlike the method of the same name on http.Server, this function defaults to +// enforcing TLS 1.0 or higher in order to address the POODLE vulnerability. +// Users who wish to enable SSLv3 must do so by supplying a TLSConfig +// explicitly. func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { addr := srv.Addr if addr == "" { addr = ":https" } - config := &tls.Config{} + config := &tls.Config{ + MinVersion: tls.VersionTLS10, + } if srv.TLSConfig != nil { *config = *srv.TLSConfig } @@ -68,7 +74,11 @@ func ListenAndServe(addr string, handler http.Handler) error { return server.ListenAndServe() } -// ListenAndServeTLS behaves exactly like the net/http function of the same name. +// ListenAndServeTLS behaves almost exactly like the net/http function of the +// same name. Unlike net/http, however, this function defaults to enforcing TLS +// 1.0 or higher in order to address the POODLE vulnerability. Users who wish to +// enable SSLv3 must do so by explicitly instantiating a server with an +// appropriately configured TLSConfig property. func ListenAndServeTLS(addr, certfile, keyfile string, handler http.Handler) error { server := &Server{Addr: addr, Handler: handler} return server.ListenAndServeTLS(certfile, keyfile) From 863ba3029d64b5ec1ab68b383249dd4c044b3838 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 5 Jan 2015 19:06:32 +0100 Subject: [PATCH 168/217] Allow regexps to take advantage of SubRouter Go's regular expressions don't allow you to create a capturing group named "*", which previously made using SubRouter with regular expression patterns impossible. This change introduces the alternate key "_", which happens to be a legal capturing group name. Fixes #98. --- web/middleware/subrouter.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/web/middleware/subrouter.go b/web/middleware/subrouter.go index 60de471..1b3f3f3 100644 --- a/web/middleware/subrouter.go +++ b/web/middleware/subrouter.go @@ -13,7 +13,11 @@ type subrouter struct { func (s subrouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { if s.c.URLParams != nil { - if path, ok := s.c.URLParams["*"]; ok { + path, ok := s.c.URLParams["*"] + if !ok { + path, ok = s.c.URLParams["_"] + } + if ok { oldpath := r.URL.Path r.URL.Path = path defer func() { @@ -31,6 +35,11 @@ func (s subrouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { // middleware will help you set the request URL's Path to this unmatched suffix, // allowing you to write sub-routers with no knowledge of what routes the parent // router matches. +// +// Since Go's regular expressions do not allow you to create a capturing group +// named "*", SubRouter also accepts the string "_". For instance, to duplicate +// the semantics of the string pattern "/foo/*", you might use the regular +// expression "^/foo(?P<_>/.*)$". func SubRouter(c *web.C, h http.Handler) http.Handler { return subrouter{c, h} } From b9d27b2f3413b0135afd1aaae355bbf1e205ecdb Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 6 Jan 2015 00:15:38 +0100 Subject: [PATCH 169/217] Remove graceful.Middleware from the public API It was exposed mostly for documentation of a Go 1.2 polyfill, and really doesn't feel like it should be exposed long-term, since it hasn't done anything in frightfully long. This is a breaking change for advanced users of package graceful, for which there is unfortunately no easy workaround (at least for Go 1.2 users, which are the ~only ones affected). --- graceful/middleware.go | 27 +++------------------------ graceful/middleware13.go | 12 ------------ graceful/serve.go | 2 +- graceful/serve13.go | 6 ++---- 4 files changed, 6 insertions(+), 41 deletions(-) delete mode 100644 graceful/middleware13.go diff --git a/graceful/middleware.go b/graceful/middleware.go index 7f1371e..94edfe3 100644 --- a/graceful/middleware.go +++ b/graceful/middleware.go @@ -12,30 +12,9 @@ import ( "github.com/zenazn/goji/graceful/listener" ) -/* -Middleware adds graceful shutdown capabilities to the given handler. When a -graceful shutdown is in progress, this middleware intercepts responses to add a -"Connection: close" header to politely inform the client that we are about to go -away. - -This package creates a shim http.ResponseWriter that it passes to subsequent -handlers. Unfortunately, there's a great many optional interfaces that this -http.ResponseWriter might implement (e.g., http.CloseNotifier, http.Flusher, and -http.Hijacker), and in order to perfectly proxy all of these options we'd be -left with some kind of awful powerset of ResponseWriters, and that's not even -counting all the other custom interfaces you might be expecting. Instead of -doing that, we have implemented two kinds of proxies: one that contains no -additional methods (i.e., exactly corresponding to the http.ResponseWriter -interface), and one that supports all three of http.CloseNotifier, http.Flusher, -and http.Hijacker. If you find that this is not enough, the original -http.ResponseWriter can be retrieved by calling Unwrap() on the proxy object. - -This middleware is automatically applied to every http.Handler passed to this -package, and most users will not need to call this function directly. It is -exported primarily for documentation purposes and in the off chance that someone -really wants more control over their http.Server than we currently provide. -*/ -func Middleware(h http.Handler) http.Handler { +// Middleware provides functionality similar to net/http.Server's +// SetKeepAlivesEnabled in Go 1.3, but in Go 1.2. +func middleware(h http.Handler) http.Handler { if h == nil { return nil } diff --git a/graceful/middleware13.go b/graceful/middleware13.go deleted file mode 100644 index b1a7e2d..0000000 --- a/graceful/middleware13.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build go1.3 - -package graceful - -import "net/http" - -// Middleware is a stub that does nothing. When used with versions of Go before -// Go 1.3, it provides functionality similar to net/http.Server's -// SetKeepAlivesEnabled. -func Middleware(h http.Handler) http.Handler { - return h -} diff --git a/graceful/serve.go b/graceful/serve.go index f4ed19f..567fb8e 100644 --- a/graceful/serve.go +++ b/graceful/serve.go @@ -22,7 +22,7 @@ func (srv *Server) Serve(l net.Listener) error { if shadow.ReadTimeout == 0 { shadow.ReadTimeout = forever } - shadow.Handler = Middleware(shadow.Handler) + shadow.Handler = middleware(shadow.Handler) wrap := listener.Wrap(l, listener.Deadline) appendListener(wrap) diff --git a/graceful/serve13.go b/graceful/serve13.go index 5f5e4ed..e9beb43 100644 --- a/graceful/serve13.go +++ b/graceful/serve13.go @@ -47,13 +47,11 @@ func (c connState) Wrap(nc net.Conn, s http.ConnState) { switch s { case http.StateIdle: if err := listener.MarkIdle(nc); err != nil { - log.Printf("error marking conn as idle: %v", - err) + log.Printf("error marking conn as idle: %v", err) } case http.StateHijacked: if err := listener.Disown(nc); err != nil { - log.Printf("error disowning hijacked conn: %v", - err) + log.Printf("error disowning hijacked conn: %v", err) } } if c != nil { From de6bd515eaef0a5c05fc04cdb3853df4f98c7c13 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 6 Jan 2015 00:40:33 +0100 Subject: [PATCH 170/217] Revert "Force use of fallback chanpool on App Engine" This has been fixed in v1.9.17 of App Engine: https://code.google.com/p/googleappengine/issues/detail?id=11346 This reverts commit 9556e1736d72993303989f72b3fef2450d121d92. Fixes #67. --- web/chanpool.go | 2 +- web/cpool.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/chanpool.go b/web/chanpool.go index 26b675e..fbe2977 100644 --- a/web/chanpool.go +++ b/web/chanpool.go @@ -1,4 +1,4 @@ -// +build !go1.3 appengine +// +build !go1.3 package web diff --git a/web/cpool.go b/web/cpool.go index a95333c..59f8764 100644 --- a/web/cpool.go +++ b/web/cpool.go @@ -1,4 +1,4 @@ -// +build go1.3,!appengine +// +build go1.3 package web From b85318d656e330037f82ada413c801128cf3def1 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 6 Jan 2015 00:52:44 +0100 Subject: [PATCH 171/217] Fix tests on Go 1.2 --- graceful/middleware_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graceful/middleware_test.go b/graceful/middleware_test.go index fe336f4..aa4e623 100644 --- a/graceful/middleware_test.go +++ b/graceful/middleware_test.go @@ -19,7 +19,7 @@ func (f fakeWriter) Write(buf []byte) (int, error) { func (f fakeWriter) WriteHeader(status int) {} func testClose(t *testing.T, h http.Handler, expectClose bool) { - m := Middleware(h) + m := middleware(h) r, _ := http.NewRequest("GET", "/", nil) w := make(fakeWriter) m.ServeHTTP(w, r) From 1af24b8302d8ec9429784bf198e0c82cc6de85c7 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 6 Jan 2015 13:20:37 +0100 Subject: [PATCH 172/217] Comment formatting nit --- graceful/serve13.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/graceful/serve13.go b/graceful/serve13.go index e9beb43..b11f10a 100644 --- a/graceful/serve13.go +++ b/graceful/serve13.go @@ -41,9 +41,9 @@ func (c connState) Wrap(nc net.Conn, s http.ConnState) { // there. // // I started a golang-nuts thread about this here: - // https://groups.google.com/forum/#!topic/golang-nuts/Xi8yjBGWfCQ I'd - // be very eager to find a better way to do this, so reach out to me if - // you have any ideas. + // https://groups.google.com/forum/#!topic/golang-nuts/Xi8yjBGWfCQ + // I'd be very eager to find a better way to do this, so reach out to me + // if you have any ideas. switch s { case http.StateIdle: if err := listener.MarkIdle(nc); err != nil { From c1922ab1807c016f4d596a9a3e65ba2433c55825 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 6 Jan 2015 15:50:31 +0100 Subject: [PATCH 173/217] Track all connections, not just idle ones This change makes package graceful/listener track all connections that have been Accepted, not just the ones considered idle. The motivation here is to support a DrainAll function on graceful listeners, allowing users an alternative to patiently waiting (potentially forever) for in-use connections to become idle. --- graceful/listener/conn.go | 18 +++++----- graceful/listener/listener.go | 19 ++++++++-- graceful/listener/listener_test.go | 13 +++++++ graceful/listener/shard.go | 57 ++++++++++++++++++++++-------- graceful/listener/shard_test.go | 21 +++++++++++ 5 files changed, 102 insertions(+), 26 deletions(-) create mode 100644 graceful/listener/shard_test.go diff --git a/graceful/listener/conn.go b/graceful/listener/conn.go index 22e986f..3f797c5 100644 --- a/graceful/listener/conn.go +++ b/graceful/listener/conn.go @@ -25,7 +25,7 @@ var errClosing = errors.New("use of closed network connection") func (c *conn) init() error { c.shard.wg.Add(1) - if shouldExit := c.shard.markIdle(c); shouldExit { + if shouldExit := c.shard.track(c); shouldExit { c.Close() return errClosing } @@ -65,12 +65,14 @@ func (c *conn) Close() error { c.mu.Lock() defer c.mu.Unlock() - if !c.closed && !c.disowned { - c.closed = true - c.shard.markInUse(c) - defer c.shard.wg.Done() + if c.closed || c.disowned { + return errClosing } + c.closed = true + c.shard.disown(c) + defer c.shard.wg.Done() + return c.Conn.Close() } @@ -98,7 +100,7 @@ func (c *conn) markIdle() { if exit := c.shard.markIdle(c); exit && !c.closed && !c.disowned { c.closed = true - c.shard.markInUse(c) + c.shard.disown(c) defer c.shard.wg.Done() c.Conn.Close() return @@ -121,7 +123,7 @@ func (c *conn) closeIfIdle() error { if !c.busy && !c.closed && !c.disowned { c.closed = true - c.shard.markInUse(c) + c.shard.disown(c) defer c.shard.wg.Done() return c.Conn.Close() } @@ -139,7 +141,7 @@ func (c *conn) disown() error { return errAlreadyDisowned } - c.shard.markInUse(c) + c.shard.disown(c) c.disowned = true c.shard.wg.Done() diff --git a/graceful/listener/listener.go b/graceful/listener/listener.go index bebfb31..1a0e59e 100644 --- a/graceful/listener/listener.go +++ b/graceful/listener/listener.go @@ -108,7 +108,7 @@ func (t *T) Close() error { // serving requests. func (t *T) CloseIdle() error { for i := range t.shards { - t.shards[i].closeIdle(false) + t.shards[i].closeConns(false, false) } // Not sure if returning errors is actually useful here :/ return nil @@ -122,7 +122,7 @@ func (t *T) CloseIdle() error { // connections will be accepted and immediately closed. func (t *T) Drain() error { for i := range t.shards { - t.shards[i].closeIdle(true) + t.shards[i].closeConns(false, true) } for i := range t.shards { t.shards[i].wait() @@ -130,7 +130,20 @@ func (t *T) Drain() error { return nil } -var notManagedErr = errors.New("listener: passed net.Conn is not managed by us") +// DrainAll closes all connections currently tracked by this listener (both idle +// and in-use connections), and prevents new connections from being accepted. +// Disowned connections are not closed. +func (t *T) DrainAll() error { + for i := range t.shards { + t.shards[i].closeConns(true, true) + } + for i := range t.shards { + t.shards[i].wait() + } + return nil +} + +var notManagedErr = errors.New("listener: passed net.Conn is not managed by this package") // Disown causes a connection to no longer be tracked by the listener. The // passed connection must have been returned by a call to Accept from this diff --git a/graceful/listener/listener_test.go b/graceful/listener/listener_test.go index 160434b..dcefdaa 100644 --- a/graceful/listener/listener_test.go +++ b/graceful/listener/listener_test.go @@ -110,6 +110,19 @@ func TestDrain(t *testing.T) { } } +func TestDrainAll(t *testing.T) { + t.Parallel() + l, c, wc := singleConn(t, Manual) + + MarkInUse(wc) + if err := l.DrainAll(); err != nil { + t.Fatalf("error draining listener: %v", err) + } + if !c.Closed() { + t.Error("expected in-use connection to be closed") + } +} + func TestErrors(t *testing.T) { t.Parallel() _, c, wc := singleConn(t, Manual) diff --git a/graceful/listener/shard.go b/graceful/listener/shard.go index e47deac..a9addad 100644 --- a/graceful/listener/shard.go +++ b/graceful/listener/shard.go @@ -6,15 +6,10 @@ type shard struct { l *T mu sync.Mutex - set map[*conn]struct{} + idle map[*conn]struct{} + all map[*conn]struct{} wg sync.WaitGroup drain bool - - // We pack shards together in an array, but we don't want them packed - // too closely, since we want to give each shard a dedicated CPU cache - // line. This amount of padding works out well for the common case of - // x64 processors (64-bit pointers with a 64-byte cache line). - _ [12]byte } // We pretty aggressively preallocate set entries in the hopes that we never @@ -25,7 +20,27 @@ const prealloc = 2048 func (s *shard) init(l *T) { s.l = l - s.set = make(map[*conn]struct{}, prealloc) + s.idle = make(map[*conn]struct{}, prealloc) + s.all = make(map[*conn]struct{}, prealloc) +} + +func (s *shard) track(c *conn) (shouldClose bool) { + s.mu.Lock() + if s.drain { + s.mu.Unlock() + return true + } + s.all[c] = struct{}{} + s.idle[c] = struct{}{} + s.mu.Unlock() + return false +} + +func (s *shard) disown(c *conn) { + s.mu.Lock() + delete(s.all, c) + delete(s.idle, c) + s.mu.Unlock() } func (s *shard) markIdle(c *conn) (shouldClose bool) { @@ -34,35 +49,47 @@ func (s *shard) markIdle(c *conn) (shouldClose bool) { s.mu.Unlock() return true } - s.set[c] = struct{}{} + s.idle[c] = struct{}{} s.mu.Unlock() return false } func (s *shard) markInUse(c *conn) { s.mu.Lock() - delete(s.set, c) + delete(s.idle, c) s.mu.Unlock() } -func (s *shard) closeIdle(drain bool) { +func (s *shard) closeConns(all, drain bool) { s.mu.Lock() if drain { s.drain = true } - set := s.set - s.set = make(map[*conn]struct{}, prealloc) + set := make(map[*conn]struct{}, len(s.all)) + if all { + for c := range s.all { + set[c] = struct{}{} + } + } else { + for c := range s.idle { + set[c] = struct{}{} + } + } // We have to drop the shard lock here to avoid deadlock: we cannot // acquire the shard lock after the connection lock, and the closeIfIdle // call below will grab a connection lock. s.mu.Unlock() - for conn := range set { + for c := range set { // This might return an error (from Close), but I don't think we // can do anything about it, so let's just pretend it didn't // happen. (I also expect that most errors returned in this way // are going to be pretty boring) - conn.closeIfIdle() + if all { + c.Close() + } else { + c.closeIfIdle() + } } } diff --git a/graceful/listener/shard_test.go b/graceful/listener/shard_test.go new file mode 100644 index 0000000..9b99394 --- /dev/null +++ b/graceful/listener/shard_test.go @@ -0,0 +1,21 @@ +// +build amd64 + +package listener + +import ( + "testing" + "unsafe" +) + +// We pack shards together in an array, but we don't want them packed too +// closely, since we want to give each shard a dedicated CPU cache line. This +// test checks this property for x64 (which has a 64-byte cache line), which +// probably covers the majority of deployments. +// +// As always, this is probably a premature optimization. +func TestShardSize(t *testing.T) { + s := unsafe.Sizeof(shard{}) + if s < 64 { + t.Errorf("sizeof(shard) = %d; expected >64", s) + } +} From 1230cef0603ee8fdbfe44eabef1fc9ff94a7eef4 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 6 Jan 2015 15:55:43 +0100 Subject: [PATCH 174/217] Reorganize code in package graceful Exactly the same code, just (hopefully) organized a little more sensibly. --- graceful/graceful.go | 104 +++++++++++-------------------------------- graceful/server.go | 82 ++++++++++++++++++++++++++++++++++ graceful/signal.go | 27 ----------- 3 files changed, 109 insertions(+), 104 deletions(-) create mode 100644 graceful/server.go diff --git a/graceful/graceful.go b/graceful/graceful.go index 255f24c..19384ff 100644 --- a/graceful/graceful.go +++ b/graceful/graceful.go @@ -8,88 +8,12 @@ automatic support for graceful restarts/code upgrades. package graceful import ( - "crypto/tls" "net" - "net/http" + "sync/atomic" "github.com/zenazn/goji/graceful/listener" ) -// Most of the code here is lifted straight from net/http - -// Type Server is exactly the same as an http.Server, but provides more graceful -// implementations of its methods. -type Server http.Server - -func (srv *Server) ListenAndServe() error { - addr := srv.Addr - if addr == "" { - addr = ":http" - } - l, e := net.Listen("tcp", addr) - if e != nil { - return e - } - return srv.Serve(l) -} - -// Unlike the method of the same name on http.Server, this function defaults to -// enforcing TLS 1.0 or higher in order to address the POODLE vulnerability. -// Users who wish to enable SSLv3 must do so by supplying a TLSConfig -// explicitly. -func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { - addr := srv.Addr - if addr == "" { - addr = ":https" - } - config := &tls.Config{ - MinVersion: tls.VersionTLS10, - } - if srv.TLSConfig != nil { - *config = *srv.TLSConfig - } - if config.NextProtos == nil { - config.NextProtos = []string{"http/1.1"} - } - - var err error - config.Certificates = make([]tls.Certificate, 1) - config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return err - } - - conn, err := net.Listen("tcp", addr) - if err != nil { - return err - } - - tlsListener := tls.NewListener(conn, config) - return srv.Serve(tlsListener) -} - -// ListenAndServe behaves exactly like the net/http function of the same name. -func ListenAndServe(addr string, handler http.Handler) error { - server := &Server{Addr: addr, Handler: handler} - return server.ListenAndServe() -} - -// ListenAndServeTLS behaves almost exactly like the net/http function of the -// same name. Unlike net/http, however, this function defaults to enforcing TLS -// 1.0 or higher in order to address the POODLE vulnerability. Users who wish to -// enable SSLv3 must do so by explicitly instantiating a server with an -// appropriately configured TLSConfig property. -func ListenAndServeTLS(addr, certfile, keyfile string, handler http.Handler) error { - server := &Server{Addr: addr, Handler: handler} - return server.ListenAndServeTLS(certfile, keyfile) -} - -// Serve behaves exactly like the net/http function of the same name. -func Serve(l net.Listener, handler http.Handler) error { - server := &Server{Handler: handler} - return server.Serve(l) -} - // WrapListener wraps an arbitrary net.Listener for use with graceful shutdowns. // In the background, it uses the listener sub-package to Wrap the listener in // Deadline mode. If another mode of operation is desired, you should call @@ -105,3 +29,29 @@ func WrapListener(l net.Listener) net.Listener { appendListener(lt) return lt } + +func appendListener(l *listener.T) { + mu.Lock() + defer mu.Unlock() + + listeners = append(listeners, l) +} + +const errClosing = "use of closed network connection" + +// During graceful shutdown, calls to Accept will start returning errors. This +// is inconvenient, since we know these sorts of errors are peaceful, so we +// silently swallow them. +func peacefulError(err error) error { + if atomic.LoadInt32(&closing) == 0 { + return err + } + // Unfortunately Go doesn't really give us a better way to select errors + // than this, so *shrug*. + if oe, ok := err.(*net.OpError); ok { + if oe.Op == "accept" && oe.Err.Error() == errClosing { + return nil + } + } + return err +} diff --git a/graceful/server.go b/graceful/server.go new file mode 100644 index 0000000..ab69236 --- /dev/null +++ b/graceful/server.go @@ -0,0 +1,82 @@ +package graceful + +import ( + "crypto/tls" + "net" + "net/http" +) + +// Most of the code here is lifted straight from net/http + +// Type Server is exactly the same as an http.Server, but provides more graceful +// implementations of its methods. +type Server http.Server + +func (srv *Server) ListenAndServe() error { + addr := srv.Addr + if addr == "" { + addr = ":http" + } + l, e := net.Listen("tcp", addr) + if e != nil { + return e + } + return srv.Serve(l) +} + +// Unlike the method of the same name on http.Server, this function defaults to +// enforcing TLS 1.0 or higher in order to address the POODLE vulnerability. +// Users who wish to enable SSLv3 must do so by supplying a TLSConfig +// explicitly. +func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { + addr := srv.Addr + if addr == "" { + addr = ":https" + } + config := &tls.Config{ + MinVersion: tls.VersionTLS10, + } + if srv.TLSConfig != nil { + *config = *srv.TLSConfig + } + if config.NextProtos == nil { + config.NextProtos = []string{"http/1.1"} + } + + var err error + config.Certificates = make([]tls.Certificate, 1) + config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return err + } + + conn, err := net.Listen("tcp", addr) + if err != nil { + return err + } + + tlsListener := tls.NewListener(conn, config) + return srv.Serve(tlsListener) +} + +// ListenAndServe behaves exactly like the net/http function of the same name. +func ListenAndServe(addr string, handler http.Handler) error { + server := &Server{Addr: addr, Handler: handler} + return server.ListenAndServe() +} + +// ListenAndServeTLS behaves almost exactly like the net/http function of the +// same name. Unlike net/http, however, this function defaults to enforcing TLS +// 1.0 or higher in order to address the POODLE vulnerability. Users who wish to +// enable SSLv3 must do so by explicitly instantiating a server with an +// appropriately configured TLSConfig property. +func ListenAndServeTLS(addr, certfile, keyfile string, handler http.Handler) error { + server := &Server{Addr: addr, Handler: handler} + return server.ListenAndServeTLS(certfile, keyfile) +} + +// Serve behaves exactly like the net/http function of the same name. +func Serve(l net.Listener, handler http.Handler) error { + server := &Server{Handler: handler} + return server.Serve(l) +} diff --git a/graceful/signal.go b/graceful/signal.go index 6bd3d84..2787476 100644 --- a/graceful/signal.go +++ b/graceful/signal.go @@ -1,7 +1,6 @@ package graceful import ( - "net" "os" "os/signal" "sync" @@ -117,29 +116,3 @@ func waitForSignal() { func Wait() { <-wait } - -func appendListener(l *listener.T) { - mu.Lock() - defer mu.Unlock() - - listeners = append(listeners, l) -} - -const errClosing = "use of closed network connection" - -// During graceful shutdown, calls to Accept will start returning errors. This -// is inconvenient, since we know these sorts of errors are peaceful, so we -// silently swallow them. -func peacefulError(err error) error { - if atomic.LoadInt32(&closing) == 0 { - return err - } - // Unfortunately Go doesn't really give us a better way to select errors - // than this, so *shrug*. - if oe, ok := err.(*net.OpError); ok { - if oe.Op == "accept" && oe.Err.Error() == errClosing { - return nil - } - } - return err -} From 3b2ab247aa6cd50b7c3fe966065901fc928ac980 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 6 Jan 2015 16:03:21 +0100 Subject: [PATCH 175/217] Get rid of many appengine build tags This is now controlled at the top level goji package: these packages are not even compiled in when using App Engine. This is sensible, since neither package is of any use there. --- bind/einhorn.go | 2 +- bind/einhorn_stub.go | 2 +- bind/systemd.go | 2 +- bind/systemd_stub.go | 2 +- graceful/einhorn.go | 2 +- serve_appengine.go | 11 ++++++++++- 6 files changed, 15 insertions(+), 6 deletions(-) diff --git a/bind/einhorn.go b/bind/einhorn.go index d18b694..e695c0e 100644 --- a/bind/einhorn.go +++ b/bind/einhorn.go @@ -1,4 +1,4 @@ -// +build !windows,!appengine +// +build !windows package bind diff --git a/bind/einhorn_stub.go b/bind/einhorn_stub.go index 18cd3b3..093707f 100644 --- a/bind/einhorn_stub.go +++ b/bind/einhorn_stub.go @@ -1,4 +1,4 @@ -// +build windows appengine +// +build windows package bind diff --git a/bind/systemd.go b/bind/systemd.go index 292f509..e7cd8e4 100644 --- a/bind/systemd.go +++ b/bind/systemd.go @@ -1,4 +1,4 @@ -// +build !windows,!appengine +// +build !windows package bind diff --git a/bind/systemd_stub.go b/bind/systemd_stub.go index 1ac1bad..4ad4d20 100644 --- a/bind/systemd_stub.go +++ b/bind/systemd_stub.go @@ -1,4 +1,4 @@ -// +build windows appengine +// +build windows package bind diff --git a/graceful/einhorn.go b/graceful/einhorn.go index ad78d1c..082d1c4 100644 --- a/graceful/einhorn.go +++ b/graceful/einhorn.go @@ -1,4 +1,4 @@ -// +build !windows,!appengine +// +build !windows package graceful diff --git a/serve_appengine.go b/serve_appengine.go index 2d39665..88dc7a8 100644 --- a/serve_appengine.go +++ b/serve_appengine.go @@ -2,7 +2,16 @@ package goji -import "net/http" +import ( + "log" + "net/http" +) + +func init() { + if fl := log.Flags(); fl&log.Ltime != 0 { + log.SetFlags(fl | log.Lmicroseconds) + } +} // Serve starts Goji using reasonable defaults. func Serve() { From 624e61b87915f107493cc7966c91841b3f2c324f Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 6 Jan 2015 17:22:01 +0100 Subject: [PATCH 176/217] Expose graceful.ShutdownNow This backends to the listener.DrainAll functionality I just added, allowing impatient users to shut down their services immediately and un-gracefully if they want. --- graceful/signal.go | 111 +++++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 39 deletions(-) diff --git a/graceful/signal.go b/graceful/signal.go index 2787476..97a2832 100644 --- a/graceful/signal.go +++ b/graceful/signal.go @@ -19,10 +19,6 @@ var wait = make(chan struct{}) var stdSignals = []os.Signal{os.Interrupt} var sigchan = make(chan os.Signal, 1) -func init() { - go waitForSignal() -} - // HandleSignals installs signal handlers for a set of standard signals. By // default, this set only includes keyboard interrupts, however when the package // detects that it is running under Einhorn, a SIGUSR2 handler is installed as @@ -42,20 +38,6 @@ func ResetSignals() { signal.Stop(sigchan) } -type userShutdown struct{} - -func (u userShutdown) String() string { - return "application initiated shutdown" -} -func (u userShutdown) Signal() {} - -// Shutdown manually triggers a shutdown from your application. Like Wait(), -// blocks until all connections have gracefully shut down. -func Shutdown() { - sigchan <- userShutdown{} - <-wait -} - // PreHook registers a function to be called before any of this package's normal // shutdown actions. All listeners will be called in the order they were added, // from a single goroutine. @@ -81,38 +63,89 @@ func PostHook(f func()) { posthooks = append(posthooks, f) } -func waitForSignal() { - <-sigchan +// Shutdown manually triggers a shutdown from your application. Like Wait, +// blocks until all connections have gracefully shut down. +func Shutdown() { + shutdown(false) +} - mu.Lock() - defer mu.Unlock() +// ShutdownNow triggers an immediate shutdown from your application. All +// connections (not just those that are idle) are immediately closed, even if +// they are in the middle of serving a request. +func ShutdownNow() { + shutdown(true) +} + +// Wait for all connections to gracefully shut down. This is commonly called at +// the bottom of the main() function to prevent the program from exiting +// prematurely. +func Wait() { + <-wait +} + +func init() { + go sigLoop() +} +func sigLoop() { + for { + <-sigchan + go shutdown(false) + } +} - for _, f := range prehooks { - f() +var preOnce, closeOnce, forceOnce, postOnce, notifyOnce sync.Once + +func shutdown(force bool) { + preOnce.Do(func() { + mu.Lock() + defer mu.Unlock() + for _, f := range prehooks { + f() + } + }) + + if force { + forceOnce.Do(func() { + closeListeners(force) + }) + } else { + closeOnce.Do(func() { + closeListeners(force) + }) } + postOnce.Do(func() { + mu.Lock() + defer mu.Unlock() + for _, f := range posthooks { + f() + } + }) + + notifyOnce.Do(func() { + close(wait) + }) +} + +func closeListeners(force bool) { atomic.StoreInt32(&closing, 1) + var wg sync.WaitGroup + defer wg.Wait() + + mu.Lock() + defer mu.Unlock() wg.Add(len(listeners)) + for _, l := range listeners { go func(l *listener.T) { defer wg.Done() l.Close() - l.Drain() + if force { + l.DrainAll() + } else { + l.Drain() + } }(l) } - wg.Wait() - - for _, f := range posthooks { - f() - } - - close(wait) -} - -// Wait for all connections to gracefully shut down. This is commonly called at -// the bottom of the main() function to prevent the program from exiting -// prematurely. -func Wait() { - <-wait } From e2c32e0616b174a7e885c678b795edc1203a81c6 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 6 Jan 2015 17:44:00 +0100 Subject: [PATCH 177/217] Add graceful.DoubleKickWindow This allows users to signal Goji to exit ungracefully by means of sending two signals in quick succession. --- graceful/signal.go | 24 +++++++++++++++++++++++- serve.go | 2 ++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/graceful/signal.go b/graceful/signal.go index 97a2832..e93e2bd 100644 --- a/graceful/signal.go +++ b/graceful/signal.go @@ -5,6 +5,7 @@ import ( "os/signal" "sync" "sync/atomic" + "time" "github.com/zenazn/goji/graceful/listener" ) @@ -14,6 +15,7 @@ var listeners = make([]*listener.T, 0) var prehooks = make([]func(), 0) var posthooks = make([]func(), 0) var closing int32 +var doubleKick, timeout time.Duration var wait = make(chan struct{}) var stdSignals = []os.Signal{os.Interrupt} @@ -76,6 +78,20 @@ func ShutdownNow() { shutdown(true) } +// DoubleKickWindow sets the length of the window during which two back-to-back +// signals are treated as an especially urgent or forceful request to exit +// (i.e., ShutdownNow instead of Shutdown). Signals delivered more than this +// duration apart are treated as separate requests to exit gracefully as usual. +func DoubleKickWindow(d time.Duration) { + if d < 0 { + return + } + mu.Lock() + defer mu.Unlock() + + doubleKick = d +} + // Wait for all connections to gracefully shut down. This is commonly called at // the bottom of the main() function to prevent the program from exiting // prematurely. @@ -87,9 +103,15 @@ func init() { go sigLoop() } func sigLoop() { + var last time.Time for { <-sigchan - go shutdown(false) + now := time.Now() + mu.Lock() + force := doubleKick != 0 && now.Sub(last) < doubleKick + mu.Unlock() + go shutdown(force) + last = now } } diff --git a/serve.go b/serve.go index cb4e405..d90e818 100644 --- a/serve.go +++ b/serve.go @@ -6,6 +6,7 @@ import ( "flag" "log" "net/http" + "time" "github.com/zenazn/goji/bind" "github.com/zenazn/goji/graceful" @@ -16,6 +17,7 @@ func init() { if fl := log.Flags(); fl&log.Ltime != 0 { log.SetFlags(fl | log.Lmicroseconds) } + graceful.DoubleKickWindow(2 * time.Second) } // Serve starts Goji using reasonable defaults. From dcb9598cbbcfcc10d2c1ee1e66958e54a2c84b52 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 6 Jan 2015 17:55:52 +0100 Subject: [PATCH 178/217] Add graceful.Timeout This allows users to escalate a graceful shutdown to a forceful one if there are active connections that are taking too long to finish their requests. Fixes #94. --- graceful/signal.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/graceful/signal.go b/graceful/signal.go index e93e2bd..60612b8 100644 --- a/graceful/signal.go +++ b/graceful/signal.go @@ -82,6 +82,8 @@ func ShutdownNow() { // signals are treated as an especially urgent or forceful request to exit // (i.e., ShutdownNow instead of Shutdown). Signals delivered more than this // duration apart are treated as separate requests to exit gracefully as usual. +// +// Setting DoubleKickWindow to 0 disables the feature. func DoubleKickWindow(d time.Duration) { if d < 0 { return @@ -92,6 +94,22 @@ func DoubleKickWindow(d time.Duration) { doubleKick = d } +// Timeout sets the maximum amount of time package graceful will wait for +// connections to gracefully shut down after receiving a signal. After this +// timeout, connections will be forcefully shut down (similar to calling +// ShutdownNow). +// +// Setting Timeout to 0 disables the feature. +func Timeout(d time.Duration) { + if d < 0 { + return + } + mu.Lock() + defer mu.Unlock() + + timeout = d +} + // Wait for all connections to gracefully shut down. This is commonly called at // the bottom of the main() function to prevent the program from exiting // prematurely. @@ -109,6 +127,12 @@ func sigLoop() { now := time.Now() mu.Lock() force := doubleKick != 0 && now.Sub(last) < doubleKick + if t := timeout; t != 0 && !force { + go func() { + time.Sleep(t) + shutdown(true) + }() + } mu.Unlock() go shutdown(force) last = now From 9148477cfd68c2ff962a03bcc6059bebaf2fa07a Mon Sep 17 00:00:00 2001 From: zhibinr Date: Tue, 20 Jan 2015 19:34:57 +0800 Subject: [PATCH 179/217] miss a "s" in the URL --- example/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/main.go b/example/main.go index 8d6b634..9ea2b0f 100644 --- a/example/main.go +++ b/example/main.go @@ -131,7 +131,7 @@ func GetUser(c web.C, w http.ResponseWriter, r *http.Request) { } } -// GetGreet finds a particular greet by ID (GET "/greet/\d+"). Does no bounds +// GetGreet finds a particular greet by ID (GET "/greets/\d+"). Does no bounds // checking, so will probably panic. func GetGreet(c web.C, w http.ResponseWriter, r *http.Request) { id, err := strconv.Atoi(c.URLParams["id"]) From 1e119c31799f7a26844db00ed0a73893a2d046be Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 24 Jan 2015 17:29:47 -0800 Subject: [PATCH 180/217] I guess it's 2015 now --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 777b8f4..128f3fc 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2014 Carl Jackson (carl@avtok.com) +Copyright (c) 2014, 2015 Carl Jackson (carl@avtok.com) MIT License From 845982030542a0fd9c8e8cbf76a84c7482cb3755 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 18 Jan 2015 15:31:22 -0800 Subject: [PATCH 181/217] Make C.Env a map[interface{}]interface{} This is a major breaking change to web.C, the Goji context object. The Env key has been changed from a map[string]interface{} to a map[interface{}]interface{} in order to better support package-private environment values and to make the context value more compatible with golang.org/x/net/context's Context. Since strings support equality, most existing uses of web.C should continue to function. Users who construct Env by hand (i.e., by calling "make") will need to update their code as instructed by their compiler. Users who iterate over the environment will need to update their code to take into account the fact that keys may no longer be strings. --- web/middleware/envinit.go | 31 +++++------ web/middleware/options_test.go | 2 +- web/middleware/realip.go | 14 +---- web/middleware/request_id.go | 2 +- web/middleware_test.go | 2 +- web/mux_test.go | 2 +- web/pattern.go | 4 +- web/router.go | 2 +- web/router_test.go | 2 +- web/web.go | 94 +++++++++++++++------------------- 10 files changed, 67 insertions(+), 88 deletions(-) diff --git a/web/middleware/envinit.go b/web/middleware/envinit.go index aefefb7..ae3b683 100644 --- a/web/middleware/envinit.go +++ b/web/middleware/envinit.go @@ -6,21 +6,22 @@ import ( "github.com/zenazn/goji/web" ) -// EnvInit is a middleware that allocates an environment map if one does not -// already exist. This is necessary because Goji does not guarantee that Env is -// present when running middleware (it avoids forcing the map allocation). Note -// that other middleware should check Env for nil in order to maximize -// compatibility (when EnvInit is not used, or when another middleware layer -// blanks out Env), but for situations in which the user controls the middleware -// stack and knows EnvInit is present, this middleware can eliminate a lot of -// boilerplate. -func EnvInit(c *web.C, h http.Handler) http.Handler { - fn := func(w http.ResponseWriter, r *http.Request) { - if c.Env == nil { - c.Env = make(map[string]interface{}) - } - h.ServeHTTP(w, r) +type envInit struct { + c *web.C + h http.Handler +} + +func (e envInit) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if e.c.Env == nil { + e.c.Env = make(map[interface{}]interface{}) } + e.h.ServeHTTP(w, r) +} - return http.HandlerFunc(fn) +// EnvInit is a middleware that allocates an environment map if it is nil. While +// it's impossible in general to ensure that Env is never nil in a middleware +// stack, in most common cases placing this middleware at the top of the stack +// will eliminate the need for repetative nil checks. +func EnvInit(c *web.C, h http.Handler) http.Handler { + return envInit{c, h} } diff --git a/web/middleware/options_test.go b/web/middleware/options_test.go index 74f5481..e962145 100644 --- a/web/middleware/options_test.go +++ b/web/middleware/options_test.go @@ -21,7 +21,7 @@ func testOptions(r *http.Request, f func(*web.C, http.ResponseWriter, *http.Requ return w } -var optionsTestEnv = map[string]interface{}{ +var optionsTestEnv = map[interface{}]interface{}{ web.ValidMethodsKey: []string{ "hello", "world", diff --git a/web/middleware/realip.go b/web/middleware/realip.go index 99cd311..ae5599f 100644 --- a/web/middleware/realip.go +++ b/web/middleware/realip.go @@ -3,20 +3,14 @@ package middleware import ( "net/http" "strings" - - "github.com/zenazn/goji/web" ) -// Key the original value of RemoteAddr is stored under. -const OriginalRemoteAddrKey = "originalRemoteAddr" - var xForwardedFor = http.CanonicalHeaderKey("X-Forwarded-For") var xRealIP = http.CanonicalHeaderKey("X-Real-IP") // RealIP is a middleware that sets a http.Request's RemoteAddr to the results // of parsing either the X-Forwarded-For header or the X-Real-IP header (in that -// order). It places the original value of RemoteAddr in a context environment -// variable. +// order). // // This middleware should be inserted fairly early in the middleware stack to // ensure that subsequent layers (e.g., request loggers) which examine the @@ -29,13 +23,9 @@ var xRealIP = http.CanonicalHeaderKey("X-Real-IP") // values from the client, or if you use this middleware without a reverse // proxy, malicious clients will be able to make you very sad (or, depending on // how you're using RemoteAddr, vulnerable to an attack of some sort). -func RealIP(c *web.C, h http.Handler) http.Handler { +func RealIP(h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { if rip := realIP(r); rip != "" { - if c.Env == nil { - c.Env = make(map[string]interface{}) - } - c.Env[OriginalRemoteAddrKey] = r.RemoteAddr r.RemoteAddr = rip } h.ServeHTTP(w, r) diff --git a/web/middleware/request_id.go b/web/middleware/request_id.go index 27e2313..834d8e3 100644 --- a/web/middleware/request_id.go +++ b/web/middleware/request_id.go @@ -60,7 +60,7 @@ func init() { func RequestID(c *web.C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { if c.Env == nil { - c.Env = make(map[string]interface{}) + c.Env = make(map[interface{}]interface{}) } myid := atomic.AddUint64(&reqid, 1) c.Env[RequestIDKey] = fmt.Sprintf("%s-%06d", prefix, myid) diff --git a/web/middleware_test.go b/web/middleware_test.go index b262b07..678ccaa 100644 --- a/web/middleware_test.go +++ b/web/middleware_test.go @@ -179,7 +179,7 @@ func TestContext(t *testing.T) { if c.Env != nil || c.URLParams != nil { t.Error("Expected a clean context") } - c.Env = make(map[string]interface{}) + c.Env = make(map[interface{}]interface{}) c.Env["reqID"] = 1 h.ServeHTTP(w, r) diff --git a/web/mux_test.go b/web/mux_test.go index 1854524..f461452 100644 --- a/web/mux_test.go +++ b/web/mux_test.go @@ -36,7 +36,7 @@ func TestIfItWorks(t *testing.T) { } r, _ = http.NewRequest("GET", "/hello/bob", nil) - env := map[string]interface{}{"greeting": "Yo "} + env := map[interface{}]interface{}{"greeting": "Yo "} m.ServeHTTPC(C{Env: env}, httptest.NewRecorder(), r) out = <-ch if out != "Yo bob" { diff --git a/web/pattern.go b/web/pattern.go index 99ca59e..c277ae1 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -56,7 +56,9 @@ of three types: that is anchored on the left (i.e., the beginning of the string). If your regular expression is not anchored on the left, a hopefully-identical left-anchored regular expression will be created - and used instead. + and used instead. Named capturing groups will bind URLParams of the + same name; unnamed capturing groups will be bound to the variables + "$1", "$2", etc. ParsePattern fatally exits (using log.Fatalf) if it is passed a value of an unexpected type. It is the caller's responsibility to ensure that ParsePattern diff --git a/web/router.go b/web/router.go index c291294..4659d85 100644 --- a/web/router.go +++ b/web/router.go @@ -129,7 +129,7 @@ func (rt *router) route(c *C, w http.ResponseWriter, r *http.Request) { sort.Strings(methodsList) if c.Env == nil { - c.Env = map[string]interface{}{ + c.Env = map[interface{}]interface{}{ ValidMethodsKey: methodsList, } } else { diff --git a/web/router_test.go b/web/router_test.go index 44dabf5..1d2c27e 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -310,7 +310,7 @@ func TestValidMethods(t *testing.T) { // This should also work when c.Env has already been initalized m.Use(func(c *C, h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - c.Env = make(map[string]interface{}) + c.Env = make(map[interface{}]interface{}) h.ServeHTTP(w, r) }) }) diff --git a/web/web.go b/web/web.go index 480f670..81a2bae 100644 --- a/web/web.go +++ b/web/web.go @@ -1,26 +1,18 @@ /* -Package web implements a fast and flexible middleware stack and mux. - -The underlying philosophy behind this package is that net/http is a very good -HTTP library which is only missing a few features. If you disagree with this -statement (e.g., you think that the interfaces it exposes are not especially -good, or if you're looking for a comprehensive "batteries included" feature -list), you're likely not going to have a good time using this library. In that -spirit, we have attempted wherever possible to be compatible with net/http. You -should be able to insert any net/http compliant handler into this library, or -use this library with any other net/http compliant mux. +Package web provides a fast and flexible middleware stack and mux. This package attempts to solve three problems that net/http does not. First, it -allows you to specify URL patterns with Sinatra-like named wildcards and -regexps. Second, it allows you to write reconfigurable middleware stacks. And -finally, it allows you to attach additional context to requests, in a manner -that can be manipulated by both compliant middleware and handlers. +allows you to specify flexible patterns, including routes with named parameters +and regular expressions. Second, it allows you to write reconfigurable +middleware stacks. And finally, it allows you to attach additional context to +requests, in a manner that can be manipulated by both compliant middleware and +handlers. A usage example: m := web.New() -Use your favorite HTTP verbs: +Use your favorite HTTP verbs and the interfaces you know and love from net/http: var legacyFooHttpHandler http.Handler // From elsewhere m.Get("/foo", legacyFooHttpHandler) @@ -28,7 +20,7 @@ Use your favorite HTTP verbs: w.Write([]byte("Hello world!")) }) -Bind parameters using either Sinatra-like patterns or regular expressions: +Bind parameters using either named captures or regular expressions: m.Get("/hello/:name", func(c web.C, w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Hello, %s!", c.URLParams["name"]) @@ -41,33 +33,32 @@ Bind parameters using either Sinatra-like patterns or regular expressions: Middleware are functions that wrap http.Handlers, just like you'd use with raw net/http. Middleware functions can optionally take a context parameter, which will be threaded throughout the middleware stack and to the final handler, even -if not all of these things support contexts. Middleware are encouraged to use -the Env parameter to pass data to other middleware and to the final handler: +if not all of the intervening middleware or handlers support contexts. +Middleware are encouraged to use the Env parameter to pass request-scoped data +to other middleware and to the final handler: - m.Use(func(h http.Handler) http.Handler { + func LoggerMiddleware(h http.Handler) http.Handler { handler := func(w http.ResponseWriter, r *http.Request) { log.Println("Before request") h.ServeHTTP(w, r) log.Println("After request") } return http.HandlerFunc(handler) - }) - m.Use(func(c *web.C, h http.Handler) http.Handler { + } + func AuthMiddleware(c *web.C, h http.Handler) http.Handler { handler := func(w http.ResponseWriter, r *http.Request) { - cookie, err := r.Cookie("user") - if err == nil { - // Consider using the middleware EnvInit instead - // of repeating the below check - if c.Env == nil { - c.Env = make(map[string]interface{}) - } + if cookie, err := r.Cookie("user"); err == nil { c.Env["user"] = cookie.Value } h.ServeHTTP(w, r) } return http.HandlerFunc(handler) - }) + } + // This makes the AuthMiddleware above a little cleaner + m.Use(middleware.EnvInit) + m.Use(AuthMiddleware) + m.Use(LoggerMiddleware) m.Get("/baz", func(c web.C, w http.ResponseWriter, r *http.Request) { if user, ok := c.Env["user"].(string); ok { w.Write([]byte("Hello " + user)) @@ -83,46 +74,41 @@ import ( ) /* -C is a per-request context object which is threaded through all compliant middleware -layers and to the final request handler. - -As an implementation detail, references to these structs are reused between -requests to reduce allocation churn, but the maps they contain are created fresh -on every request. If you are closing over a context (especially relevant for -middleware), you should not close over either the URLParams or Env objects, -instead accessing them through the context whenever they are required. +C is a request-local context object which is threaded through all compliant +middleware layers and given to the final request handler. */ type C struct { - // The parameters parsed by the mux from the URL itself. In most cases, - // will contain a map from programmer-specified identifiers to the - // strings that matched those identifiers, but if a unnamed regex - // capture is used, it will be assigned to the special identifiers "$1", - // "$2", etc. + // URLParams is a map of variables extracted from the URL (typically + // from the path portion) during routing. See the documentation for the + // URL Pattern you are using (or the documentation for ParsePattern for + // the case of standard pattern types) for more information about how + // variables are extracted and named. URLParams map[string]string - // A free-form environment, similar to Rack or PEP 333's environments. - // Middleware layers are encouraged to pass data to downstream layers - // and other handlers using this map, and are even more strongly - // encouraged to document and maybe namespace the keys they use. - Env map[string]interface{} + // Env is a free-form environment for storing request-local data. Keys + // may be arbitrary types that support equality, however package-private + // types with type-safe accessors provide a convenient way for packages + // to mediate access to their request-local data. + Env map[interface{}]interface{} } -// Handler is a superset of net/http's http.Handler, which also includes a -// mechanism for serving requests with a context. If your handler does not -// support the use of contexts, we encourage you to use http.Handler instead. +// Handler is similar to net/http's http.Handler, but also accepts a Goji +// context object. type Handler interface { - http.Handler ServeHTTPC(C, http.ResponseWriter, *http.Request) } -// HandlerFunc is like net/http's http.HandlerFunc, but supports a context -// object. Implements both http.Handler and web.Handler free of charge. +// HandlerFunc is similar to net/http's http.HandlerFunc, but supports a context +// object. Implements both http.Handler and web.Handler. type HandlerFunc func(C, http.ResponseWriter, *http.Request) +// ServeHTTP implements http.Handler, allowing HandlerFunc's to be used with +// net/http and other compliant routers. When used in this way, the underlying +// function will be passed an empty context. func (h HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { h(C{}, w, r) } -// ServeHTTPC wraps ServeHTTP with a context parameter. +// ServeHTTPC implements Handler. func (h HandlerFunc) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { h(c, w, r) } From d65ed21ea90ebb3b630eebc16efffb7a932432fe Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 15 Feb 2015 16:07:02 -0800 Subject: [PATCH 182/217] Use "vanity" types to clarify use of interface{} In order to expose a convenient API, it's unfortunately necessary to lean on Go's interface{} quite a bit: in reality we only accept a handful of types at each call site, but it's impossible to express this using the type system. Prior to this commit (as well as the ParsePattern commit), I exposed all of this type information in the form of an enormous comment on web.Mux, however this was pretty gross. Instead, let's use "vanity" type aliases for interface{} to provide documentation about which types are accepted where. This will hopefully make the API documentation easier to skim, but doesn't affect any existing uses of Goji. This commit also clarifies a couple other parts of the documentation. --- default.go | 28 ++++++------ web/mux.go | 119 ++++++++++++++++++------------------------------- web/pattern.go | 31 ++----------- web/web.go | 61 ++++++++++++++++++++++++- 4 files changed, 120 insertions(+), 119 deletions(-) diff --git a/default.go b/default.go index 541830e..540e792 100644 --- a/default.go +++ b/default.go @@ -19,84 +19,84 @@ func init() { // Use appends the given middleware to the default Mux's middleware stack. See // the documentation for web.Mux.Use for more information. -func Use(middleware interface{}) { +func Use(middleware web.MiddlewareType) { DefaultMux.Use(middleware) } // Insert the given middleware into the default Mux's middleware stack. See the // documentation for web.Mux.Insert for more information. -func Insert(middleware, before interface{}) error { +func Insert(middleware, before web.MiddlewareType) error { return DefaultMux.Insert(middleware, before) } // Abandon removes the given middleware from the default Mux's middleware stack. // See the documentation for web.Mux.Abandon for more information. -func Abandon(middleware interface{}) error { +func Abandon(middleware web.MiddlewareType) error { return DefaultMux.Abandon(middleware) } // Handle adds a route to the default Mux. See the documentation for web.Mux for // more information about what types this function accepts. -func Handle(pattern interface{}, handler interface{}) { +func Handle(pattern web.PatternType, handler web.HandlerType) { DefaultMux.Handle(pattern, handler) } // Connect adds a CONNECT route to the default Mux. See the documentation for // web.Mux for more information about what types this function accepts. -func Connect(pattern interface{}, handler interface{}) { +func Connect(pattern web.PatternType, handler web.HandlerType) { DefaultMux.Connect(pattern, handler) } // Delete adds a DELETE route to the default Mux. See the documentation for // web.Mux for more information about what types this function accepts. -func Delete(pattern interface{}, handler interface{}) { +func Delete(pattern web.PatternType, handler web.HandlerType) { DefaultMux.Delete(pattern, handler) } // Get adds a GET route to the default Mux. See the documentation for web.Mux for // more information about what types this function accepts. -func Get(pattern interface{}, handler interface{}) { +func Get(pattern web.PatternType, handler web.HandlerType) { DefaultMux.Get(pattern, handler) } // Head adds a HEAD route to the default Mux. See the documentation for web.Mux // for more information about what types this function accepts. -func Head(pattern interface{}, handler interface{}) { +func Head(pattern web.PatternType, handler web.HandlerType) { DefaultMux.Head(pattern, handler) } // Options adds a OPTIONS route to the default Mux. See the documentation for // web.Mux for more information about what types this function accepts. -func Options(pattern interface{}, handler interface{}) { +func Options(pattern web.PatternType, handler web.HandlerType) { DefaultMux.Options(pattern, handler) } // Patch adds a PATCH route to the default Mux. See the documentation for web.Mux // for more information about what types this function accepts. -func Patch(pattern interface{}, handler interface{}) { +func Patch(pattern web.PatternType, handler web.HandlerType) { DefaultMux.Patch(pattern, handler) } // Post adds a POST route to the default Mux. See the documentation for web.Mux // for more information about what types this function accepts. -func Post(pattern interface{}, handler interface{}) { +func Post(pattern web.PatternType, handler web.HandlerType) { DefaultMux.Post(pattern, handler) } // Put adds a PUT route to the default Mux. See the documentation for web.Mux for // more information about what types this function accepts. -func Put(pattern interface{}, handler interface{}) { +func Put(pattern web.PatternType, handler web.HandlerType) { DefaultMux.Put(pattern, handler) } // Trace adds a TRACE route to the default Mux. See the documentation for // web.Mux for more information about what types this function accepts. -func Trace(pattern interface{}, handler interface{}) { +func Trace(pattern web.PatternType, handler web.HandlerType) { DefaultMux.Trace(pattern, handler) } // NotFound sets the NotFound handler for the default Mux. See the documentation // for web.Mux.NotFound for more information. -func NotFound(handler interface{}) { +func NotFound(handler web.HandlerType) { DefaultMux.NotFound(handler) } diff --git a/web/mux.go b/web/mux.go index 3908de5..c48ed4b 100644 --- a/web/mux.go +++ b/web/mux.go @@ -5,38 +5,18 @@ import ( ) /* -Mux is an HTTP multiplexer, much like net/http's ServeMux. - -Routes may be added using any of the various HTTP-method-specific functions. -When processing a request, when iterating in insertion order the first route -that matches both the request's path and method is used. - -There are two other differences worth mentioning between web.Mux and -http.ServeMux. First, string patterns (i.e., Sinatra-like patterns) must match -exactly: the "rooted subtree" behavior of ServeMux is not implemented. Secondly, -unlike ServeMux, Mux does not support Host-specific patterns. - -If you require any of these features, remember that you are free to mix and -match muxes at any part of the stack. - -In order to provide a sane API, many functions on Mux take interface{}'s. This -is obviously not a very satisfying solution, but it's probably the best we can -do for now. Instead of duplicating documentation on each method, the types -accepted by those functions are documented here. - -A middleware (the untyped parameter in Use() and Insert()) must be one of the -following types: - - func(http.Handler) http.Handler - - func(c *web.C, http.Handler) http.Handler - -All of the route-adding functions on Mux take two untyped parameters: pattern -and handler. Pattern will be passed to ParsePattern, which takes a web.Pattern, -a string, or a regular expression (more information can be found in the -ParsePattern documentation). Handler must be one of the following types: - - http.Handler - - web.Handler - - func(w http.ResponseWriter, r *http.Request) - - func(c web.C, w http.ResponseWriter, r *http.Request) +Mux is an HTTP multiplexer, much like net/http's ServeMux. It functions as both +a middleware stack and as an HTTP router. + +Middleware provide a great abstraction for actions that must be performed on +every request, such as request logging and authentication. To append, insert, +and remove middleware, you can call the Use, Insert, and Abandon functions +respectively. + +Routes may be added using any of the HTTP verb functions (Get, Post, etc.), or +through the generic Handle function. Goji's routing algorithm is very simple: +routes are processed in the order they are added, and the first matching route +will be executed. Routes match if their HTTP method and Pattern both match. */ type Mux struct { ms mStack @@ -59,7 +39,7 @@ func New() *Mux { return &mux } -// ServeHTTP processes HTTP requests. It make Muxes satisfy net/http.Handler. +// ServeHTTP processes HTTP requests. Satisfies net/http.Handler. func (m *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) { stack := m.ms.alloc() stack.ServeHTTP(w, r) @@ -67,7 +47,7 @@ func (m *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // ServeHTTPC creates a context dependent request with the given Mux. Satisfies -// the web.Handler interface. +// the Handler interface. func (m *Mux) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { stack := m.ms.alloc() stack.ServeHTTPC(c, w, r) @@ -76,23 +56,21 @@ func (m *Mux) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { // Middleware Stack functions -// Append the given middleware to the middleware stack. See the documentation -// for type Mux for a list of valid middleware types. +// Append the given middleware to the middleware stack. // // No attempt is made to enforce the uniqueness of middlewares. It is illegal to // call this function concurrently with active requests. -func (m *Mux) Use(middleware interface{}) { +func (m *Mux) Use(middleware MiddlewareType) { m.ms.Use(middleware) } // Insert the given middleware immediately before a given existing middleware in -// the stack. See the documentation for type Mux for a list of valid middleware -// types. Returns an error if no middleware has the name given by "before." +// the stack. Returns an error if "before" cannot be found in the current stack. // // No attempt is made to enforce the uniqueness of middlewares. If the insertion // point is ambiguous, the first (outermost) one is chosen. It is illegal to // call this function concurrently with active requests. -func (m *Mux) Insert(middleware, before interface{}) error { +func (m *Mux) Insert(middleware, before MiddlewareType) error { return m.ms.Insert(middleware, before) } @@ -102,7 +80,7 @@ func (m *Mux) Insert(middleware, before interface{}) error { // If the name of the middleware to delete is ambiguous, the first (outermost) // one is chosen. It is illegal to call this function concurrently with active // requests. -func (m *Mux) Abandon(middleware interface{}) error { +func (m *Mux) Abandon(middleware MiddlewareType) error { return m.ms.Abandon(middleware) } @@ -110,96 +88,85 @@ func (m *Mux) Abandon(middleware interface{}) error { /* Dispatch to the given handler when the pattern matches, regardless of HTTP -method. See the documentation for type Mux for a description of what types are -accepted for pattern and handler. +method. This method is commonly used to implement sub-routing: an admin application, for instance, can expose a single handler that is attached to the main Mux by -calling Handle("/admin*", adminHandler) or similar. Note that this function +calling Handle("/admin/*", adminHandler) or similar. Note that this function doesn't strip this prefix from the path before forwarding it on (e.g., the -handler will see the full path, including the "/admin" part), but this +handler will see the full path, including the "/admin/" part), but this functionality can easily be performed by an extra middleware layer. */ -func (m *Mux) Handle(pattern interface{}, handler interface{}) { +func (m *Mux) Handle(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mALL, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is -// CONNECT. See the documentation for type Mux for a description of what types -// are accepted for pattern and handler. -func (m *Mux) Connect(pattern interface{}, handler interface{}) { +// CONNECT. +func (m *Mux) Connect(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mCONNECT, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is -// DELETE. See the documentation for type Mux for a description of what types -// are accepted for pattern and handler. -func (m *Mux) Delete(pattern interface{}, handler interface{}) { +// DELETE. +func (m *Mux) Delete(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mDELETE, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is -// GET. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. +// GET. // // All GET handlers also transparently serve HEAD requests, since net/http will // take care of all the fiddly bits for you. If you wish to provide an alternate // implementation of HEAD, you should add a handler explicitly and place it // above your GET handler. -func (m *Mux) Get(pattern interface{}, handler interface{}) { +func (m *Mux) Get(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mGET|mHEAD, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is -// HEAD. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. -func (m *Mux) Head(pattern interface{}, handler interface{}) { +// HEAD. +func (m *Mux) Head(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mHEAD, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is -// OPTIONS. See the documentation for type Mux for a description of what types -// are accepted for pattern and handler. -func (m *Mux) Options(pattern interface{}, handler interface{}) { +// OPTIONS. +func (m *Mux) Options(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mOPTIONS, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is -// PATCH. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. -func (m *Mux) Patch(pattern interface{}, handler interface{}) { +// PATCH. +func (m *Mux) Patch(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mPATCH, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is -// POST. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. -func (m *Mux) Post(pattern interface{}, handler interface{}) { +// POST. +func (m *Mux) Post(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mPOST, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is -// PUT. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. -func (m *Mux) Put(pattern interface{}, handler interface{}) { +// PUT. +func (m *Mux) Put(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mPUT, handler) } // Dispatch to the given handler when the pattern matches and the HTTP method is -// TRACE. See the documentation for type Mux for a description of what types are -// accepted for pattern and handler. -func (m *Mux) Trace(pattern interface{}, handler interface{}) { +// TRACE. +func (m *Mux) Trace(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mTRACE, handler) } -// Set the fallback (i.e., 404) handler for this mux. See the documentation for -// type Mux for a description of what types are accepted for handler. +// Set the fallback (i.e., 404) handler for this mux. // // As a convenience, the context environment variable "goji.web.validMethods" // (also available as the constant ValidMethodsKey) will be set to the list of // HTTP methods that could have been routed had they been provided on an // otherwise identical request. -func (m *Mux) NotFound(handler interface{}) { +func (m *Mux) NotFound(handler HandlerType) { m.rt.notFound = parseHandler(handler) } diff --git a/web/pattern.go b/web/pattern.go index c277ae1..3b64385 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -36,35 +36,12 @@ ParsePattern is used internally by Goji to parse route patterns. It is exposed publicly to make it easier to write thin wrappers around the built-in Pattern implementations. -Although its parameter has type interface{}, ParsePattern only accepts arguments -of three types: - - web.Pattern, which is passed through - - string, which is interpreted as a Sinatra-like URL pattern. In - particular, the following syntax is recognized: - - a path segment starting with with a colon will match any - string placed at that position. e.g., "/:name" will match - "/carl", binding "name" to "carl". - - a pattern ending with "/*" will match any route with that - prefix. For instance, the pattern "/u/:name/*" will match - "/u/carl/" and "/u/carl/projects/123", but not "/u/carl" - (because there is no trailing slash). In addition to any names - bound in the pattern, the special key "*" is bound to the - unmatched tail of the match, but including the leading "/". So - for the two matching examples above, "*" would be bound to "/" - and "/projects/123" respectively. - - regexp.Regexp, which is assumed to be a Perl-style regular expression - that is anchored on the left (i.e., the beginning of the string). If - your regular expression is not anchored on the left, a - hopefully-identical left-anchored regular expression will be created - and used instead. Named capturing groups will bind URLParams of the - same name; unnamed capturing groups will be bound to the variables - "$1", "$2", etc. - ParsePattern fatally exits (using log.Fatalf) if it is passed a value of an -unexpected type. It is the caller's responsibility to ensure that ParsePattern -is called in a type-safe manner. +unexpected type (see the documentation for PatternType for a list of which types +are accepted). It is the caller's responsibility to ensure that ParsePattern is +called in a type-safe manner. */ -func ParsePattern(raw interface{}) Pattern { +func ParsePattern(raw PatternType) Pattern { switch v := raw.(type) { case Pattern: return v diff --git a/web/web.go b/web/web.go index 81a2bae..95b25a4 100644 --- a/web/web.go +++ b/web/web.go @@ -80,7 +80,7 @@ middleware layers and given to the final request handler. type C struct { // URLParams is a map of variables extracted from the URL (typically // from the path portion) during routing. See the documentation for the - // URL Pattern you are using (or the documentation for ParsePattern for + // URL Pattern you are using (or the documentation for PatternType for // the case of standard pattern types) for more information about how // variables are extracted and named. URLParams map[string]string @@ -98,7 +98,7 @@ type Handler interface { } // HandlerFunc is similar to net/http's http.HandlerFunc, but supports a context -// object. Implements both http.Handler and web.Handler. +// object. Implements both http.Handler and Handler. type HandlerFunc func(C, http.ResponseWriter, *http.Request) // ServeHTTP implements http.Handler, allowing HandlerFunc's to be used with @@ -112,3 +112,60 @@ func (h HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h HandlerFunc) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { h(c, w, r) } + +/* +PatternType is the type denoting Patterns and types that Goji internally +converts to Pattern (via the ParsePattern function). In order to provide an +expressive API, this type is an alias for interface{} (that is named for the +purposes of documentation), however only the following concrete types are +accepted: + - types that implement Pattern + - string, which is interpreted as a Sinatra-like URL pattern. In + particular, the following syntax is recognized: + - a path segment starting with with a colon will match any + string placed at that position. e.g., "/:name" will match + "/carl", binding "name" to "carl". + - a pattern ending with "/*" will match any route with that + prefix. For instance, the pattern "/u/:name/*" will match + "/u/carl/" and "/u/carl/projects/123", but not "/u/carl" + (because there is no trailing slash). In addition to any names + bound in the pattern, the special key "*" is bound to the + unmatched tail of the match, but including the leading "/". So + for the two matching examples above, "*" would be bound to "/" + and "/projects/123" respectively. + Unlike http.ServeMux's patterns, string patterns support neither the + "rooted subtree" behavior nor Host-specific routes. Users who require + either of these features are encouraged to compose package http's mux + with the mux provided by this package. + - regexp.Regexp, which is assumed to be a Perl-style regular expression + that is anchored on the left (i.e., the beginning of the string). If + your regular expression is not anchored on the left, a + hopefully-identical left-anchored regular expression will be created + and used instead. + + Capturing groups will be converted into bound URL parameters in + URLParams. If the capturing group is named, that name will be used; + otherwise the special identifiers "$1", "$2", etc. will be used. +*/ +type PatternType interface{} + +/* +HandlerType is the type of Handlers and types that Goji internally converts to +Handler. In order to provide an expressive API, this type is an alias for +interface{} (that is named for the purposes of documentation), however only the +following concrete types are accepted: + - types that implement http.Handler + - types that implement Handler + - func(w http.ResponseWriter, r *http.Request) + - func(c web.C, w http.ResponseWriter, r *http.Request) +*/ +type HandlerType interface{} + +/* +MiddlewareType is the type of Goji middleware. In order to provide an expressive +API, this type is an alias for interface{} (that is named for the purposes of +documentation), however only the following concrete types are accepted: + - func(http.Handler) http.Handler + - func(c *web.C, http.Handler) http.Handler +*/ +type MiddlewareType interface{} From 7558aee4ad25f82597f06807563b00bcd5d33980 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 15 Feb 2015 16:21:16 -0800 Subject: [PATCH 183/217] Link to godoc in type-check error messages This beats trying to enumerate the accepted types by hand. --- web/middleware.go | 6 +++--- web/pattern.go | 7 ++++--- web/router.go | 9 ++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/web/middleware.go b/web/middleware.go index 21e000b..7ec545d 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -50,6 +50,8 @@ func (s *cStack) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { s.m.ServeHTTP(w, r) } +const unknownMiddleware = `Unknown middleware type %T. See http://godoc.org/github.com/zenazn/goji/web#MiddlewareType for a list of acceptable types.` + func (m *mStack) appendLayer(fn interface{}) { ml := mLayer{orig: fn} switch f := fn.(type) { @@ -60,9 +62,7 @@ func (m *mStack) appendLayer(fn interface{}) { case func(*C, http.Handler) http.Handler: ml.fn = f default: - log.Fatalf(`Unknown middleware type %v. Expected a function `+ - `with signature "func(http.Handler) http.Handler" or `+ - `"func(*web.C, http.Handler) http.Handler".`, fn) + log.Fatalf(unknownMiddleware, fn) } m.stack = append(m.stack, ml) } diff --git a/web/pattern.go b/web/pattern.go index 3b64385..9f9fc85 100644 --- a/web/pattern.go +++ b/web/pattern.go @@ -31,6 +31,8 @@ type Pattern interface { Run(r *http.Request, c *C) } +const unknownPattern = `Unknown pattern type %T. See http://godoc.org/github.com/zenazn/goji/web#PatternType for a list of acceptable types.` + /* ParsePattern is used internally by Goji to parse route patterns. It is exposed publicly to make it easier to write thin wrappers around the built-in Pattern @@ -50,8 +52,7 @@ func ParsePattern(raw PatternType) Pattern { case string: return parseStringPattern(v) default: - log.Fatalf("Unknown pattern type %T. Expected a web.Pattern, "+ - "regexp.Regexp, or a string.", v) + log.Fatalf(unknownPattern, v) + panic("log.Fatalf does not return") } - panic("log.Fatalf does not return") } diff --git a/web/router.go b/web/router.go index 4659d85..f2dd6b5 100644 --- a/web/router.go +++ b/web/router.go @@ -66,6 +66,8 @@ func (h netHTTPWrap) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { h.Handler.ServeHTTP(w, r) } +const unknownHandler = `Unknown handler type %T. See http://godoc.org/github.com/zenazn/goji/web#HandlerType for a list of acceptable types.` + func parseHandler(h interface{}) Handler { switch f := h.(type) { case Handler: @@ -77,12 +79,9 @@ func parseHandler(h interface{}) Handler { case func(w http.ResponseWriter, r *http.Request): return netHTTPWrap{http.HandlerFunc(f)} default: - log.Fatalf("Unknown handler type %v. Expected a web.Handler, "+ - "a http.Handler, or a function with signature func(C, "+ - "http.ResponseWriter, *http.Request) or "+ - "func(http.ResponseWriter, *http.Request)", h) + log.Fatalf(unknownHandler, h) + panic("log.Fatalf does not return") } - panic("log.Fatalf does not return") } func httpMethod(mname string) method { From 3273bcd371f9b2b4bb14ecb3ad58e11e90ea1a16 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 15 Feb 2015 16:27:49 -0800 Subject: [PATCH 184/217] Wording --- web/web.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/web.go b/web/web.go index 95b25a4..85b06bb 100644 --- a/web/web.go +++ b/web/web.go @@ -14,8 +14,8 @@ A usage example: Use your favorite HTTP verbs and the interfaces you know and love from net/http: - var legacyFooHttpHandler http.Handler // From elsewhere - m.Get("/foo", legacyFooHttpHandler) + var existingHTTPHandler http.Handler // From elsewhere + m.Get("/foo", existingHTTPHandler) m.Post("/bar", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Hello world!")) }) From a452de8605db976edad45da8ac80985c1fd6fdf5 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 15 Feb 2015 17:51:12 -0800 Subject: [PATCH 185/217] Benchmark goji/web in parallel Test with `go test -cpu`. In other news, Goji is really really fast. Who knew! --- web/bench_test.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/web/bench_test.go b/web/bench_test.go index 57ecf41..4cc7826 100644 --- a/web/bench_test.go +++ b/web/bench_test.go @@ -93,13 +93,18 @@ func benchN(b *testing.B, n int) { for _, prefix := range prefixes { addRoutes(m, prefix) } + m.Compile() reqs := permuteRequests(genRequests(prefixes)) b.ResetTimer() b.ReportAllocs() - for i := 0; i < b.N; i++ { - m.ServeHTTP(w, reqs[i%len(reqs)]) - } + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + i++ + m.ServeHTTP(w, reqs[i%len(reqs)]) + } + }) } func benchM(b *testing.B, n int) { @@ -109,18 +114,22 @@ func benchM(b *testing.B, n int) { m.Use(trivialMiddleware) } r, _ := http.NewRequest("GET", "/", nil) + m.Compile() b.ResetTimer() b.ReportAllocs() - for i := 0; i < b.N; i++ { - m.ServeHTTP(w, r) - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + m.ServeHTTP(w, r) + } + }) } func BenchmarkStatic(b *testing.B) { m := New() m.Get("/", nilRouter{}) r, _ := http.NewRequest("GET", "/", nil) + m.Compile() b.ResetTimer() b.ReportAllocs() From 3bfbfe4c5e3b71095a7e9244e6398c6b7578396e Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 15 Feb 2015 17:58:11 -0800 Subject: [PATCH 186/217] Oops! Forgot one --- web/bench_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web/bench_test.go b/web/bench_test.go index 4cc7826..8246b4b 100644 --- a/web/bench_test.go +++ b/web/bench_test.go @@ -133,9 +133,11 @@ func BenchmarkStatic(b *testing.B) { b.ResetTimer() b.ReportAllocs() - for i := 0; i < b.N; i++ { - m.ServeHTTP(w, r) - } + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + m.ServeHTTP(w, r) + } + }) } func BenchmarkRoute5(b *testing.B) { From d7045fdf646b0489b7bfd77751c8074e9ba03ad0 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 15 Feb 2015 18:39:45 -0800 Subject: [PATCH 187/217] Only benchmark on Go 1.3+ testing.PB is a 1.3-ism. --- web/bench_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/bench_test.go b/web/bench_test.go index 8246b4b..fc285db 100644 --- a/web/bench_test.go +++ b/web/bench_test.go @@ -1,3 +1,5 @@ +// +build go1.3 + package web import ( From a06d00e4f3d8c3f15ce51930d385781b6bbf011b Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 15 Feb 2015 21:00:24 -0800 Subject: [PATCH 188/217] Use native GoDoc examples This replaces the enormous free-form docstring at the top with something that's at least syntax highlighted and collapsible. I'm a little worried about discoverability, but "oh well." --- web/example_test.go | 69 +++++++++++++++++++++++++++++++++++++++++++++ web/web.go | 59 -------------------------------------- 2 files changed, 69 insertions(+), 59 deletions(-) create mode 100644 web/example_test.go diff --git a/web/example_test.go b/web/example_test.go new file mode 100644 index 0000000..9dc8de2 --- /dev/null +++ b/web/example_test.go @@ -0,0 +1,69 @@ +package web_test + +import ( + "fmt" + "log" + "net/http" + "regexp" + + "github.com/zenazn/goji/web" + "github.com/zenazn/goji/web/middleware" +) + +func Example() { + m := web.New() + + // Use your favorite HTTP verbs and the interfaces you know and love + // from net/http: + m.Get("/hello", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Why hello there!\n") + }) + m.Post("/login", func(w http.ResponseWriter, r *http.Request) { + if r.FormValue("password") != "god" { + http.Error(w, "Hack the planet!", 401) + } + }) + + // Handlers can optionally take a context parameter, which contains + // (among other things) a set of bound parameters. + hello := func(c web.C, w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello, %s!\n", c.URLParams["name"]) + } + + // Bind parameters using pattern strings... + m.Get("/hello/:name", hello) + + // ...or use regular expressions if you need additional power. + bonjour := regexp.MustCompile(`^/bonjour/(?P[A-Za-z]+)$`) + m.Get(bonjour, hello) + + // Middleware are a great abstraction for performing logic on every + // request. Some middleware use the Goji context object to set + // request-scoped variables. + logger := func(h http.Handler) http.Handler { + wrap := func(w http.ResponseWriter, r *http.Request) { + log.Println("Before request") + h.ServeHTTP(w, r) + log.Println("After request") + } + return http.HandlerFunc(wrap) + } + auth := func(c *web.C, h http.Handler) http.Handler { + wrap := func(w http.ResponseWriter, r *http.Request) { + if cookie, err := r.Cookie("user"); err == nil { + c.Env["user"] = cookie.Value + } + h.ServeHTTP(w, r) + } + return http.HandlerFunc(wrap) + } + + // A Middleware stack is a flexible way to assemble the common + // components of your application, like request loggers and + // authentication. There is an ecosystem of open-source middleware for + // Goji, so there's a chance someone has already written the middleware + // you are looking for! + m.Use(middleware.EnvInit) + m.Use(logger) + m.Use(auth) +} diff --git a/web/web.go b/web/web.go index 85b06bb..01cd7e8 100644 --- a/web/web.go +++ b/web/web.go @@ -7,65 +7,6 @@ and regular expressions. Second, it allows you to write reconfigurable middleware stacks. And finally, it allows you to attach additional context to requests, in a manner that can be manipulated by both compliant middleware and handlers. - -A usage example: - - m := web.New() - -Use your favorite HTTP verbs and the interfaces you know and love from net/http: - - var existingHTTPHandler http.Handler // From elsewhere - m.Get("/foo", existingHTTPHandler) - m.Post("/bar", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("Hello world!")) - }) - -Bind parameters using either named captures or regular expressions: - - m.Get("/hello/:name", func(c web.C, w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "Hello, %s!", c.URLParams["name"]) - }) - pattern := regexp.MustCompile(`^/ip/(?P(?:\d{1,3}\.){3}\d{1,3})$`) - m.Get(pattern, func(c web.C, w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "Info for IP address %s:", c.URLParams["ip"]) - }) - -Middleware are functions that wrap http.Handlers, just like you'd use with raw -net/http. Middleware functions can optionally take a context parameter, which -will be threaded throughout the middleware stack and to the final handler, even -if not all of the intervening middleware or handlers support contexts. -Middleware are encouraged to use the Env parameter to pass request-scoped data -to other middleware and to the final handler: - - func LoggerMiddleware(h http.Handler) http.Handler { - handler := func(w http.ResponseWriter, r *http.Request) { - log.Println("Before request") - h.ServeHTTP(w, r) - log.Println("After request") - } - return http.HandlerFunc(handler) - } - func AuthMiddleware(c *web.C, h http.Handler) http.Handler { - handler := func(w http.ResponseWriter, r *http.Request) { - if cookie, err := r.Cookie("user"); err == nil { - c.Env["user"] = cookie.Value - } - h.ServeHTTP(w, r) - } - return http.HandlerFunc(handler) - } - - // This makes the AuthMiddleware above a little cleaner - m.Use(middleware.EnvInit) - m.Use(AuthMiddleware) - m.Use(LoggerMiddleware) - m.Get("/baz", func(c web.C, w http.ResponseWriter, r *http.Request) { - if user, ok := c.Env["user"].(string); ok { - w.Write([]byte("Hello " + user)) - } else { - w.Write([]byte("Hello Stranger!")) - } - }) */ package web From 7a0c709c84f4f8562c5600d0086de2d4b21bdb8f Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 15 Feb 2015 21:23:31 -0800 Subject: [PATCH 189/217] No parameter names in documentation --- web/web.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/web.go b/web/web.go index 01cd7e8..0fd762c 100644 --- a/web/web.go +++ b/web/web.go @@ -97,8 +97,8 @@ interface{} (that is named for the purposes of documentation), however only the following concrete types are accepted: - types that implement http.Handler - types that implement Handler - - func(w http.ResponseWriter, r *http.Request) - - func(c web.C, w http.ResponseWriter, r *http.Request) + - func(http.ResponseWriter, *http.Request) + - func(web.C, http.ResponseWriter, *http.Request) */ type HandlerType interface{} @@ -107,6 +107,6 @@ MiddlewareType is the type of Goji middleware. In order to provide an expressive API, this type is an alias for interface{} (that is named for the purposes of documentation), however only the following concrete types are accepted: - func(http.Handler) http.Handler - - func(c *web.C, http.Handler) http.Handler + - func(*web.C, http.Handler) http.Handler */ type MiddlewareType interface{} From a2ec7ac6835a6af626f7a08a21bbc690502518ea Mon Sep 17 00:00:00 2001 From: Yasuhiro Matsumoto Date: Mon, 23 Feb 2015 16:34:00 +0900 Subject: [PATCH 190/217] Handle "AcceptEx" as error op on windows --- graceful/graceful.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/graceful/graceful.go b/graceful/graceful.go index 19384ff..ff9b186 100644 --- a/graceful/graceful.go +++ b/graceful/graceful.go @@ -9,6 +9,7 @@ package graceful import ( "net" + "runtime" "sync/atomic" "github.com/zenazn/goji/graceful/listener" @@ -49,7 +50,11 @@ func peacefulError(err error) error { // Unfortunately Go doesn't really give us a better way to select errors // than this, so *shrug*. if oe, ok := err.(*net.OpError); ok { - if oe.Op == "accept" && oe.Err.Error() == errClosing { + errOp := "accept" + if runtime.GOOS == "windows" { + errOp = "AcceptEx" + } + if oe.Op == errOp && oe.Err.Error() == errClosing { return nil } } From 707cf7e127e738386a38fa9fda3834315892b434 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 12 Apr 2015 15:37:38 -0700 Subject: [PATCH 191/217] Configurable routing and router middleware This change exposes a new type, Match, which represents a matched route. When present in the Goji environment (bound to the key MatchKey), routing will be skipped and the bound Match will be dispatched to instead. In addition, the Goji router has been exposed as a middleware using the Match mechanism above. This allows middleware inserted after the router access to the Match object and any bound URLParams. Fixes #76. See also #32. --- web/func_equal.go | 10 ++---- web/handler.go | 42 ++++++++++++++++++++++ web/match.go | 66 +++++++++++++++++++++++++++++++++++ web/match_test.go | 50 ++++++++++++++++++++++++++ web/mux.go | 25 +++++++++++++ web/router.go | 52 +++++++++------------------ web/router_middleware_test.go | 35 +++++++++++++++++++ 7 files changed, 237 insertions(+), 43 deletions(-) create mode 100644 web/handler.go create mode 100644 web/match.go create mode 100644 web/match_test.go create mode 100644 web/router_middleware_test.go diff --git a/web/func_equal.go b/web/func_equal.go index 3206b04..9c8f7cb 100644 --- a/web/func_equal.go +++ b/web/func_equal.go @@ -4,10 +4,6 @@ import ( "reflect" ) -func isFunc(fn interface{}) bool { - return reflect.ValueOf(fn).Kind() == reflect.Func -} - /* This is more than a little sketchtacular. Go's rules for function pointer equality are pretty restrictive: nil function pointers always compare equal, and @@ -25,12 +21,10 @@ purposes. If you're curious, you can read more about the representation of functions here: http://golang.org/s/go11func We're in effect comparing the pointers of the indirect layer. + +This function also works on non-function values. */ func funcEqual(a, b interface{}) bool { - if !isFunc(a) || !isFunc(b) { - panic("funcEqual: type error!") - } - av := reflect.ValueOf(&a).Elem() bv := reflect.ValueOf(&b).Elem() diff --git a/web/handler.go b/web/handler.go new file mode 100644 index 0000000..746c9f0 --- /dev/null +++ b/web/handler.go @@ -0,0 +1,42 @@ +package web + +import ( + "log" + "net/http" +) + +const unknownHandler = `Unknown handler type %T. See http://godoc.org/github.com/zenazn/goji/web#HandlerType for a list of acceptable types.` + +type netHTTPHandlerWrap struct{ http.Handler } +type netHTTPHandlerFuncWrap struct { + fn func(http.ResponseWriter, *http.Request) +} +type handlerFuncWrap struct { + fn func(C, http.ResponseWriter, *http.Request) +} + +func (h netHTTPHandlerWrap) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { + h.Handler.ServeHTTP(w, r) +} +func (h netHTTPHandlerFuncWrap) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { + h.fn(w, r) +} +func (h handlerFuncWrap) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { + h.fn(c, w, r) +} + +func parseHandler(h HandlerType) Handler { + switch f := h.(type) { + case func(c C, w http.ResponseWriter, r *http.Request): + return handlerFuncWrap{f} + case func(w http.ResponseWriter, r *http.Request): + return netHTTPHandlerFuncWrap{f} + case Handler: + return f + case http.Handler: + return netHTTPHandlerWrap{f} + default: + log.Fatalf(unknownHandler, h) + panic("log.Fatalf does not return") + } +} diff --git a/web/match.go b/web/match.go new file mode 100644 index 0000000..1a44144 --- /dev/null +++ b/web/match.go @@ -0,0 +1,66 @@ +package web + +// The key used to store route Matches in the Goji environment. If this key is +// present in the environment and contains a value of type Match, routing will +// not be performed, and the Match's Handler will be used instead. +const MatchKey = "goji.web.Match" + +// Match is the type of routing matches. It is inserted into C.Env under +// MatchKey when the Mux.Router middleware is invoked. If MatchKey is present at +// route dispatch time, the Handler of the corresponding Match will be called +// instead of performing routing as usual. +// +// By computing a Match and inserting it into the Goji environment as part of a +// middleware stack (see Mux.Router, for instance), it is possible to customize +// Goji's routing behavior or replace it entirely. +type Match struct { + // Pattern is the Pattern that matched during routing. Will be nil if no + // route matched (Handler will be set to the Mux's NotFound handler) + Pattern Pattern + // The Handler corresponding to the matched pattern. + Handler Handler +} + +// GetMatch returns the Match stored in the Goji environment, or an empty Match +// if none exists (valid Matches always have a Handler property). +func GetMatch(c C) Match { + if c.Env == nil { + return Match{} + } + mi, ok := c.Env[MatchKey] + if !ok { + return Match{} + } + if m, ok := mi.(Match); ok { + return m + } + return Match{} +} + +// RawPattern returns the PatternType that was originally passed to ParsePattern +// or any of the HTTP method functions (Get, Post, etc.). +func (m Match) RawPattern() PatternType { + switch v := m.Pattern.(type) { + case regexpPattern: + return v.re + case stringPattern: + return v.raw + default: + return v + } +} + +// RawHandler returns the HandlerType that was originally passed to the HTTP +// method functions (Get, Post, etc.). +func (m Match) RawHandler() HandlerType { + switch v := m.Handler.(type) { + case netHTTPHandlerWrap: + return v.Handler + case handlerFuncWrap: + return v.fn + case netHTTPHandlerFuncWrap: + return v.fn + default: + return v + } +} diff --git a/web/match_test.go b/web/match_test.go new file mode 100644 index 0000000..aefff04 --- /dev/null +++ b/web/match_test.go @@ -0,0 +1,50 @@ +package web + +import ( + "net/http" + "regexp" + "testing" +) + +var rawPatterns = []PatternType{ + "/hello/:name", + regexp.MustCompile("^/hello/(?P[^/]+)$"), + testPattern{}, +} + +func TestRawPattern(t *testing.T) { + t.Parallel() + + for _, p := range rawPatterns { + m := Match{Pattern: ParsePattern(p)} + if rp := m.RawPattern(); rp != p { + t.Errorf("got %#v, expected %#v", rp, p) + } + } +} + +type httpHandlerOnly struct{} + +func (httpHandlerOnly) ServeHTTP(w http.ResponseWriter, r *http.Request) {} + +type handlerOnly struct{} + +func (handlerOnly) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) {} + +var rawHandlers = []HandlerType{ + func(w http.ResponseWriter, r *http.Request) {}, + func(c C, w http.ResponseWriter, r *http.Request) {}, + httpHandlerOnly{}, + handlerOnly{}, +} + +func TestRawHandler(t *testing.T) { + t.Parallel() + + for _, h := range rawHandlers { + m := Match{Handler: parseHandler(h)} + if rh := m.RawHandler(); !funcEqual(rh, h) { + t.Errorf("got %#v, expected %#v", rh, h) + } + } +} diff --git a/web/mux.go b/web/mux.go index c48ed4b..228c0a0 100644 --- a/web/mux.go +++ b/web/mux.go @@ -86,6 +86,31 @@ func (m *Mux) Abandon(middleware MiddlewareType) error { // Router functions +type routerMiddleware struct { + m *Mux + c *C + h http.Handler +} + +func (rm routerMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if rm.c.Env == nil { + rm.c.Env = make(map[interface{}]interface{}, 1) + } + rm.c.Env[MatchKey] = rm.m.rt.getMatch(rm.c, w, r) + rm.h.ServeHTTP(w, r) +} + +// Router is a middleware that performs routing and stores the resulting Match +// in Goji's environment. If a routing Match is present at the end of the +// middleware stack, that Match is used instead of re-routing. +// +// This middleware is especially useful to create post-routing middleware, e.g. +// a request logger which prints which pattern or handler was selected, or an +// authentication middleware which only applies to certain routes. +func (m *Mux) Router(c *C, h http.Handler) http.Handler { + return routerMiddleware{m, c, h} +} + /* Dispatch to the given handler when the pattern matches, regardless of HTTP method. diff --git a/web/router.go b/web/router.go index f2dd6b5..1fbc41f 100644 --- a/web/router.go +++ b/web/router.go @@ -1,7 +1,6 @@ package web import ( - "log" "net/http" "sort" "strings" @@ -30,7 +29,7 @@ const ( // The key used to communicate to the NotFound handler what methods would have // been allowed if they'd been provided. -const ValidMethodsKey = "goji.web.validMethods" +const ValidMethodsKey = "goji.web.ValidMethods" var validMethodsMap = map[string]method{ "CONNECT": mCONNECT, @@ -58,32 +57,6 @@ type router struct { machine *routeMachine } -type netHTTPWrap struct { - http.Handler -} - -func (h netHTTPWrap) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { - h.Handler.ServeHTTP(w, r) -} - -const unknownHandler = `Unknown handler type %T. See http://godoc.org/github.com/zenazn/goji/web#HandlerType for a list of acceptable types.` - -func parseHandler(h interface{}) Handler { - switch f := h.(type) { - case Handler: - return f - case http.Handler: - return netHTTPWrap{f} - case func(c C, w http.ResponseWriter, r *http.Request): - return HandlerFunc(f) - case func(w http.ResponseWriter, r *http.Request): - return netHTTPWrap{http.HandlerFunc(f)} - default: - log.Fatalf(unknownHandler, h) - panic("log.Fatalf does not return") - } -} - func httpMethod(mname string) method { if method, ok := validMethodsMap[mname]; ok { return method @@ -102,7 +75,7 @@ func (rt *router) compile() *routeMachine { return &sm } -func (rt *router) route(c *C, w http.ResponseWriter, r *http.Request) { +func (rt *router) getMatch(c *C, w http.ResponseWriter, r *http.Request) Match { rm := rt.getMachine() if rm == nil { rm = rt.compile() @@ -110,13 +83,14 @@ func (rt *router) route(c *C, w http.ResponseWriter, r *http.Request) { methods, route := rm.route(c, w, r) if route != nil { - route.handler.ServeHTTPC(*c, w, r) - return + return Match{ + Pattern: route.pattern, + Handler: route.handler, + } } if methods == 0 { - rt.notFound.ServeHTTPC(*c, w, r) - return + return Match{Handler: rt.notFound} } var methodsList = make([]string, 0) @@ -134,10 +108,18 @@ func (rt *router) route(c *C, w http.ResponseWriter, r *http.Request) { } else { c.Env[ValidMethodsKey] = methodsList } - rt.notFound.ServeHTTPC(*c, w, r) + return Match{Handler: rt.notFound} +} + +func (rt *router) route(c *C, w http.ResponseWriter, r *http.Request) { + match := GetMatch(*c) + if match.Handler == nil { + match = rt.getMatch(c, w, r) + } + match.Handler.ServeHTTPC(*c, w, r) } -func (rt *router) handleUntyped(p interface{}, m method, h interface{}) { +func (rt *router) handleUntyped(p PatternType, m method, h HandlerType) { rt.handle(ParsePattern(p), m, parseHandler(h)) } diff --git a/web/router_middleware_test.go b/web/router_middleware_test.go new file mode 100644 index 0000000..d8f0a28 --- /dev/null +++ b/web/router_middleware_test.go @@ -0,0 +1,35 @@ +package web + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestRouterMiddleware(t *testing.T) { + t.Parallel() + + m := New() + ch := make(chan string, 1) + m.Get("/a", chHandler(ch, "a")) + m.Get("/b", chHandler(ch, "b")) + m.Use(m.Router) + m.Use(func(c *C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + m := GetMatch(*c) + if rp := m.RawPattern(); rp != "/a" { + t.Fatalf("RawPattern was not /a: %v", rp) + } + r.URL.Path = "/b" + h.ServeHTTP(w, r) + } + return http.HandlerFunc(fn) + }) + + r, _ := http.NewRequest("GET", "/a", nil) + w := httptest.NewRecorder() + m.ServeHTTP(w, r) + if v := <-ch; v != "a" { + t.Error("Routing was not frozen! %s", v) + } +} From aee21eb0bd4d983cc26959f10ab28d540f91f884 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 12 Apr 2015 15:58:22 -0700 Subject: [PATCH 192/217] Remove web.HandlerFunc Since Goji accepts the underlying version of this type (i.e., the raw function), and since it doesn't use web.Handler in the same way as the net/http ecosystem uses http.Handler, there's really no reason to ever use web.HandlerFunc. This is a breaking change. Developers who previously used the web.HandlerFunc type are encouraged to inline it into their own projects. --- web/router_test.go | 8 ++------ web/web.go | 16 ---------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/web/router_test.go b/web/router_test.go index 1d2c27e..ed37c6a 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -88,8 +88,7 @@ var testHandlerTable = map[string]string{ "/a": "http fn", "/b": "http handler", "/c": "web fn", - "/d": "web handler", - "/e": "httpc", + "/d": "httpc", } func TestHandlerTypes(t *testing.T) { @@ -106,10 +105,7 @@ func TestHandlerTypes(t *testing.T) { m.Get("/c", func(c C, w http.ResponseWriter, r *http.Request) { ch <- "web fn" }) - m.Get("/d", HandlerFunc(func(c C, w http.ResponseWriter, r *http.Request) { - ch <- "web handler" - })) - m.Get("/e", testHandler(ch)) + m.Get("/d", testHandler(ch)) for route, response := range testHandlerTable { r, _ := http.NewRequest("GET", route, nil) diff --git a/web/web.go b/web/web.go index 0fd762c..4394c10 100644 --- a/web/web.go +++ b/web/web.go @@ -38,22 +38,6 @@ type Handler interface { ServeHTTPC(C, http.ResponseWriter, *http.Request) } -// HandlerFunc is similar to net/http's http.HandlerFunc, but supports a context -// object. Implements both http.Handler and Handler. -type HandlerFunc func(C, http.ResponseWriter, *http.Request) - -// ServeHTTP implements http.Handler, allowing HandlerFunc's to be used with -// net/http and other compliant routers. When used in this way, the underlying -// function will be passed an empty context. -func (h HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { - h(C{}, w, r) -} - -// ServeHTTPC implements Handler. -func (h HandlerFunc) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { - h(c, w, r) -} - /* PatternType is the type denoting Patterns and types that Goji internally converts to Pattern (via the ParsePattern function). In order to provide an From 0645f4e63aa0e938a7816be673356e5c59872d51 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 12 Apr 2015 16:07:48 -0700 Subject: [PATCH 193/217] Make comments scan better --- web/web.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/web/web.go b/web/web.go index 4394c10..a65e1a2 100644 --- a/web/web.go +++ b/web/web.go @@ -41,8 +41,8 @@ type Handler interface { /* PatternType is the type denoting Patterns and types that Goji internally converts to Pattern (via the ParsePattern function). In order to provide an -expressive API, this type is an alias for interface{} (that is named for the -purposes of documentation), however only the following concrete types are +expressive API, this type is an alias for interface{} that is named for the +purposes of documentation, however only the following concrete types are accepted: - types that implement Pattern - string, which is interpreted as a Sinatra-like URL pattern. In @@ -77,7 +77,7 @@ type PatternType interface{} /* HandlerType is the type of Handlers and types that Goji internally converts to Handler. In order to provide an expressive API, this type is an alias for -interface{} (that is named for the purposes of documentation), however only the +interface{} that is named for the purposes of documentation, however only the following concrete types are accepted: - types that implement http.Handler - types that implement Handler @@ -88,8 +88,8 @@ type HandlerType interface{} /* MiddlewareType is the type of Goji middleware. In order to provide an expressive -API, this type is an alias for interface{} (that is named for the purposes of -documentation), however only the following concrete types are accepted: +API, this type is an alias for interface{} that is named for the purposes of +documentation, however only the following concrete types are accepted: - func(http.Handler) http.Handler - func(*web.C, http.Handler) http.Handler */ From 87830ef5494dd0a954ebfa242e1b1a1e85e22677 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sun, 12 Apr 2015 16:46:04 -0700 Subject: [PATCH 194/217] Fix go vet and golint errors --- graceful/listener/listener.go | 8 ++--- graceful/serve.go | 1 + graceful/serve13.go | 1 + graceful/server.go | 12 ++++--- web/bytecode_runner.go | 2 -- web/chanpool.go | 2 +- web/mux.go | 59 ++++++++++++++++++----------------- web/router_middleware_test.go | 2 +- 8 files changed, 45 insertions(+), 42 deletions(-) diff --git a/graceful/listener/listener.go b/graceful/listener/listener.go index 1a0e59e..6c9c477 100644 --- a/graceful/listener/listener.go +++ b/graceful/listener/listener.go @@ -143,7 +143,7 @@ func (t *T) DrainAll() error { return nil } -var notManagedErr = errors.New("listener: passed net.Conn is not managed by this package") +var errNotManaged = errors.New("listener: passed net.Conn is not managed by this package") // Disown causes a connection to no longer be tracked by the listener. The // passed connection must have been returned by a call to Accept from this @@ -152,7 +152,7 @@ func Disown(c net.Conn) error { if cn, ok := c.(*conn); ok { return cn.disown() } - return notManagedErr + return errNotManaged } // MarkIdle marks the given connection as being idle, and therefore eligible for @@ -163,7 +163,7 @@ func MarkIdle(c net.Conn) error { cn.markIdle() return nil } - return notManagedErr + return errNotManaged } // MarkInUse marks this connection as being in use, removing it from the set of @@ -174,5 +174,5 @@ func MarkInUse(c net.Conn) error { cn.markInUse() return nil } - return notManagedErr + return errNotManaged } diff --git a/graceful/serve.go b/graceful/serve.go index 567fb8e..edb2a53 100644 --- a/graceful/serve.go +++ b/graceful/serve.go @@ -13,6 +13,7 @@ import ( // About 200 years, also known as "forever" const forever time.Duration = 200 * 365 * 24 * time.Hour +// Serve behaves like the method on net/http.Server with the same name. func (srv *Server) Serve(l net.Listener) error { // Spawn a shadow http.Server to do the actual servering. We do this // because we need to sketch on some of the parameters you passed in, diff --git a/graceful/serve13.go b/graceful/serve13.go index b11f10a..68cac04 100644 --- a/graceful/serve13.go +++ b/graceful/serve13.go @@ -59,6 +59,7 @@ func (c connState) Wrap(nc net.Conn, s http.ConnState) { } } +// Serve behaves like the method on net/http.Server with the same name. func (srv *Server) Serve(l net.Listener) error { // Spawn a shadow http.Server to do the actual servering. We do this // because we need to sketch on some of the parameters you passed in, diff --git a/graceful/server.go b/graceful/server.go index ab69236..4176829 100644 --- a/graceful/server.go +++ b/graceful/server.go @@ -8,10 +8,11 @@ import ( // Most of the code here is lifted straight from net/http -// Type Server is exactly the same as an http.Server, but provides more graceful +// A Server is exactly the same as an http.Server, but provides more graceful // implementations of its methods. type Server http.Server +// ListenAndServe behaves like the method on net/http.Server with the same name. func (srv *Server) ListenAndServe() error { addr := srv.Addr if addr == "" { @@ -24,10 +25,11 @@ func (srv *Server) ListenAndServe() error { return srv.Serve(l) } -// Unlike the method of the same name on http.Server, this function defaults to -// enforcing TLS 1.0 or higher in order to address the POODLE vulnerability. -// Users who wish to enable SSLv3 must do so by supplying a TLSConfig -// explicitly. +// ListenAndServeTLS behaves like the method on net/http.Server with the same +// name. Unlike the method of the same name on http.Server, this function +// defaults to enforcing TLS 1.0 or higher in order to address the POODLE +// vulnerability. Users who wish to enable SSLv3 must do so by supplying a +// TLSConfig explicitly. func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { addr := srv.Addr if addr == "" { diff --git a/web/bytecode_runner.go b/web/bytecode_runner.go index 5fcf443..c32b16a 100644 --- a/web/bytecode_runner.go +++ b/web/bytecode_runner.go @@ -80,6 +80,4 @@ func (rm routeMachine) route(c *C, w http.ResponseWriter, r *http.Request) (meth i++ } } - - return methods, nil } diff --git a/web/chanpool.go b/web/chanpool.go index fbe2977..6c53c74 100644 --- a/web/chanpool.go +++ b/web/chanpool.go @@ -10,7 +10,7 @@ const cPoolSize = 32 type cPool chan *cStack func makeCPool() *cPool { - var p cPool = make(chan *cStack, cPoolSize) + p := make(cPool, cPoolSize) return &p } diff --git a/web/mux.go b/web/mux.go index 228c0a0..d7758eb 100644 --- a/web/mux.go +++ b/web/mux.go @@ -56,7 +56,7 @@ func (m *Mux) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { // Middleware Stack functions -// Append the given middleware to the middleware stack. +// Use appends the given middleware to the middleware stack. // // No attempt is made to enforce the uniqueness of middlewares. It is illegal to // call this function concurrently with active requests. @@ -64,8 +64,9 @@ func (m *Mux) Use(middleware MiddlewareType) { m.ms.Use(middleware) } -// Insert the given middleware immediately before a given existing middleware in -// the stack. Returns an error if "before" cannot be found in the current stack. +// Insert inserts the given middleware immediately before a given existing +// middleware in the stack. Returns an error if "before" cannot be found in the +// current stack. // // No attempt is made to enforce the uniqueness of middlewares. If the insertion // point is ambiguous, the first (outermost) one is chosen. It is illegal to @@ -74,8 +75,8 @@ func (m *Mux) Insert(middleware, before MiddlewareType) error { return m.ms.Insert(middleware, before) } -// Remove the given middleware from the middleware stack. Returns an error if -// no such middleware can be found. +// Abandon removes the given middleware from the middleware stack. Returns an +// error if no such middleware can be found. // // If the name of the middleware to delete is ambiguous, the first (outermost) // one is chosen. It is illegal to call this function concurrently with active @@ -112,8 +113,8 @@ func (m *Mux) Router(c *C, h http.Handler) http.Handler { } /* -Dispatch to the given handler when the pattern matches, regardless of HTTP -method. +Handle dispatches to the given handler when the pattern matches, regardless of +HTTP method. This method is commonly used to implement sub-routing: an admin application, for instance, can expose a single handler that is attached to the main Mux by @@ -126,20 +127,20 @@ func (m *Mux) Handle(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mALL, handler) } -// Dispatch to the given handler when the pattern matches and the HTTP method is -// CONNECT. +// Connect dispatches to the given handler when the pattern matches and the HTTP +// method is CONNECT. func (m *Mux) Connect(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mCONNECT, handler) } -// Dispatch to the given handler when the pattern matches and the HTTP method is -// DELETE. +// Delete dispatches to the given handler when the pattern matches and the HTTP +// method is DELETE. func (m *Mux) Delete(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mDELETE, handler) } -// Dispatch to the given handler when the pattern matches and the HTTP method is -// GET. +// Get dispatches to the given handler when the pattern matches and the HTTP +// method is GET. // // All GET handlers also transparently serve HEAD requests, since net/http will // take care of all the fiddly bits for you. If you wish to provide an alternate @@ -149,43 +150,43 @@ func (m *Mux) Get(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mGET|mHEAD, handler) } -// Dispatch to the given handler when the pattern matches and the HTTP method is -// HEAD. +// Head dispatches to the given handler when the pattern matches and the HTTP +// method is HEAD. func (m *Mux) Head(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mHEAD, handler) } -// Dispatch to the given handler when the pattern matches and the HTTP method is -// OPTIONS. +// Options dispatches to the given handler when the pattern matches and the HTTP +// method is OPTIONS. func (m *Mux) Options(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mOPTIONS, handler) } -// Dispatch to the given handler when the pattern matches and the HTTP method is -// PATCH. +// Patch dispatches to the given handler when the pattern matches and the HTTP +// method is PATCH. func (m *Mux) Patch(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mPATCH, handler) } -// Dispatch to the given handler when the pattern matches and the HTTP method is -// POST. +// Post dispatches to the given handler when the pattern matches and the HTTP +// method is POST. func (m *Mux) Post(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mPOST, handler) } -// Dispatch to the given handler when the pattern matches and the HTTP method is -// PUT. +// Put dispatches to the given handler when the pattern matches and the HTTP +// method is PUT. func (m *Mux) Put(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mPUT, handler) } -// Dispatch to the given handler when the pattern matches and the HTTP method is -// TRACE. +// Trace dispatches to the given handler when the pattern matches and the HTTP +// method is TRACE. func (m *Mux) Trace(pattern PatternType, handler HandlerType) { m.rt.handleUntyped(pattern, mTRACE, handler) } -// Set the fallback (i.e., 404) handler for this mux. +// NotFound sets the fallback (i.e., 404) handler for this mux. // // As a convenience, the context environment variable "goji.web.validMethods" // (also available as the constant ValidMethodsKey) will be set to the list of @@ -195,9 +196,9 @@ func (m *Mux) NotFound(handler HandlerType) { m.rt.notFound = parseHandler(handler) } -// Compile the list of routes into bytecode. This only needs to be done once -// after all the routes have been added, and will be called automatically for -// you (at some performance cost on the first request) if you do not call it +// Compile compiles the list of routes into bytecode. This only needs to be done +// once after all the routes have been added, and will be called automatically +// for you (at some performance cost on the first request) if you do not call it // explicitly. func (m *Mux) Compile() { m.rt.compile() diff --git a/web/router_middleware_test.go b/web/router_middleware_test.go index d8f0a28..05dc2fa 100644 --- a/web/router_middleware_test.go +++ b/web/router_middleware_test.go @@ -30,6 +30,6 @@ func TestRouterMiddleware(t *testing.T) { w := httptest.NewRecorder() m.ServeHTTP(w, r) if v := <-ch; v != "a" { - t.Error("Routing was not frozen! %s", v) + t.Errorf("Routing was not frozen! %s", v) } } From 0276f3f3fb609aabf8d9ba7aea10c80ec979989d Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 13 Apr 2015 14:44:24 -0700 Subject: [PATCH 195/217] Revert "Remove web.HandlerFunc" This reverts commit aee21eb0bd4d983cc26959f10ab28d540f91f884. Fixes #119. --- web/router_test.go | 8 ++++++-- web/web.go | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/web/router_test.go b/web/router_test.go index ed37c6a..1d2c27e 100644 --- a/web/router_test.go +++ b/web/router_test.go @@ -88,7 +88,8 @@ var testHandlerTable = map[string]string{ "/a": "http fn", "/b": "http handler", "/c": "web fn", - "/d": "httpc", + "/d": "web handler", + "/e": "httpc", } func TestHandlerTypes(t *testing.T) { @@ -105,7 +106,10 @@ func TestHandlerTypes(t *testing.T) { m.Get("/c", func(c C, w http.ResponseWriter, r *http.Request) { ch <- "web fn" }) - m.Get("/d", testHandler(ch)) + m.Get("/d", HandlerFunc(func(c C, w http.ResponseWriter, r *http.Request) { + ch <- "web handler" + })) + m.Get("/e", testHandler(ch)) for route, response := range testHandlerTable { r, _ := http.NewRequest("GET", route, nil) diff --git a/web/web.go b/web/web.go index a65e1a2..91bbedc 100644 --- a/web/web.go +++ b/web/web.go @@ -38,6 +38,22 @@ type Handler interface { ServeHTTPC(C, http.ResponseWriter, *http.Request) } +// HandlerFunc is similar to net/http's http.HandlerFunc, but supports a context +// object. Implements both http.Handler and Handler. +type HandlerFunc func(C, http.ResponseWriter, *http.Request) + +// ServeHTTP implements http.Handler, allowing HandlerFunc's to be used with +// net/http and other compliant routers. When used in this way, the underlying +// function will be passed an empty context. +func (h HandlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h(C{}, w, r) +} + +// ServeHTTPC implements Handler. +func (h HandlerFunc) ServeHTTPC(c C, w http.ResponseWriter, r *http.Request) { + h(c, w, r) +} + /* PatternType is the type denoting Patterns and types that Goji internally converts to Pattern (via the ParsePattern function). In order to provide an From 1bd09ee6902203f4453aed3206c0e471b341303d Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 30 May 2015 17:05:40 -0400 Subject: [PATCH 196/217] Allow closing of hijacked connections I think this is a regression introduced by c1922ab1. Fixes #130. --- graceful/listener/conn.go | 4 +++- graceful/listener/conn_test.go | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/graceful/listener/conn.go b/graceful/listener/conn.go index 3f797c5..7b34b47 100644 --- a/graceful/listener/conn.go +++ b/graceful/listener/conn.go @@ -65,7 +65,9 @@ func (c *conn) Close() error { c.mu.Lock() defer c.mu.Unlock() - if c.closed || c.disowned { + if c.disowned { + return c.Conn.Close() + } else if c.closed { return errClosing } diff --git a/graceful/listener/conn_test.go b/graceful/listener/conn_test.go index cd5d878..ff26e32 100644 --- a/graceful/listener/conn_test.go +++ b/graceful/listener/conn_test.go @@ -90,6 +90,22 @@ func TestCloseConn(t *testing.T) { } } +// Regression test for issue #130. +func TestDisownedClose(t *testing.T) { + t.Parallel() + _, c, wc := singleConn(t, Deadline) + + if err := Disown(wc); err != nil { + t.Fatalf("unexpected error disowning conn: %v", err) + } + if err := wc.Close(); err != nil { + t.Errorf("error closing connection: %v", err) + } + if !c.Closed() { + t.Errorf("connection didn't get closed") + } +} + func TestManualReadDeadline(t *testing.T) { t.Parallel() l, c, wc := singleConn(t, Manual) From ff41f8eaa2cfdd22e7e2f36769f9343aaee16147 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 30 May 2015 18:03:49 -0400 Subject: [PATCH 197/217] middleware: Make SubRouter Match-aware This helps avoid a gotcha when using matches with sub-routers. Fixes #120. --- web/middleware/subrouter.go | 14 ++++++++++++++ web/middleware/subrouter_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 web/middleware/subrouter_test.go diff --git a/web/middleware/subrouter.go b/web/middleware/subrouter.go index 1b3f3f3..1d371a7 100644 --- a/web/middleware/subrouter.go +++ b/web/middleware/subrouter.go @@ -19,9 +19,23 @@ func (s subrouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if ok { oldpath := r.URL.Path + oldmatch := web.GetMatch(*s.c) r.URL.Path = path + if oldmatch.Handler != nil { + delete(s.c.Env, web.MatchKey) + } + defer func() { r.URL.Path = oldpath + + if s.c.Env == nil { + return + } + if oldmatch.Handler != nil { + s.c.Env[web.MatchKey] = oldmatch + } else { + delete(s.c.Env, web.MatchKey) + } }() } } diff --git a/web/middleware/subrouter_test.go b/web/middleware/subrouter_test.go new file mode 100644 index 0000000..c46c27e --- /dev/null +++ b/web/middleware/subrouter_test.go @@ -0,0 +1,28 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/zenazn/goji/web" +) + +func TestSubRouterMatch(t *testing.T) { + m := web.New() + m.Use(m.Router) + + m2 := web.New() + m2.Use(SubRouter) + m2.Get("/bar", func(w http.ResponseWriter, r *http.Request) {}) + + m.Get("/foo/*", m2) + + r, err := http.NewRequest("GET", "/foo/bar", nil) + if err != nil { + t.Fatal(err) + } + + // This function will recurse forever if SubRouter + Match didn't work. + m.ServeHTTP(httptest.NewRecorder(), r) +} From eaa649ab77fd613bb3d44dde6b4e6043a01eee66 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 30 May 2015 18:54:29 -0400 Subject: [PATCH 198/217] Improve documentation around explicit routing Fixes #120. --- web/middleware/subrouter.go | 30 ++++++++++++++++++------------ web/mux.go | 22 +++++++++++++++------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/web/middleware/subrouter.go b/web/middleware/subrouter.go index 1d371a7..e5b0921 100644 --- a/web/middleware/subrouter.go +++ b/web/middleware/subrouter.go @@ -42,18 +42,24 @@ func (s subrouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.h.ServeHTTP(w, r) } -// SubRouter is a helper middleware that makes writing sub-routers easier. -// -// If you register a sub-router under a key like "/admin/*", Goji's router will -// automatically set c.URLParams["*"] to the unmatched path suffix. This -// middleware will help you set the request URL's Path to this unmatched suffix, -// allowing you to write sub-routers with no knowledge of what routes the parent -// router matches. -// -// Since Go's regular expressions do not allow you to create a capturing group -// named "*", SubRouter also accepts the string "_". For instance, to duplicate -// the semantics of the string pattern "/foo/*", you might use the regular -// expression "^/foo(?P<_>/.*)$". +/* +SubRouter is a helper middleware that makes writing sub-routers easier. + +If you register a sub-router under a key like "/admin/*", Goji's router will +automatically set c.URLParams["*"] to the unmatched path suffix. This middleware +will help you set the request URL's Path to this unmatched suffix, allowing you +to write sub-routers with no knowledge of what routes the parent router matches. + +Since Go's regular expressions do not allow you to create a capturing group +named "*", SubRouter also accepts the string "_". For instance, to duplicate the +semantics of the string pattern "/foo/*", you might use the regular expression +"^/foo(?P<_>/.*)$". + +This middleware is Match-aware: it will un-set any explicit routing information +contained in the Goji context in order to prevent routing loops when using +explicit routing with sub-routers. See the documentation for Mux.Router for +more. +*/ func SubRouter(c *web.C, h http.Handler) http.Handler { return subrouter{c, h} } diff --git a/web/mux.go b/web/mux.go index d7758eb..18b9991 100644 --- a/web/mux.go +++ b/web/mux.go @@ -101,13 +101,21 @@ func (rm routerMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) { rm.h.ServeHTTP(w, r) } -// Router is a middleware that performs routing and stores the resulting Match -// in Goji's environment. If a routing Match is present at the end of the -// middleware stack, that Match is used instead of re-routing. -// -// This middleware is especially useful to create post-routing middleware, e.g. -// a request logger which prints which pattern or handler was selected, or an -// authentication middleware which only applies to certain routes. +/* +Router is a middleware that performs routing and stores the resulting Match in +Goji's environment. If a routing Match is present at the end of the middleware +stack, that Match is used instead of re-routing. + +This middleware is especially useful to create post-routing middleware, e.g. a +request logger which prints which pattern or handler was selected, or an +authentication middleware which only applies to certain routes. + +If you use nested Muxes with explicit routing, you should be aware that the +explicit routing information set by an outer Mux can be picked up by an inner +Mux, inadvertently causing an infinite routing loop. If you use both explicit +routing and nested Muxes, you should be sure to unset MatchKey before the inner +Mux performs routing (or attach a Router to the inner Mux as well). +*/ func (m *Mux) Router(c *C, h http.Handler) http.Handler { return routerMiddleware{m, c, h} } From 6c0d19f81080fd4ae43ab5a0b9c9cb38ac10c78c Mon Sep 17 00:00:00 2001 From: Taichi Sasaki Date: Tue, 9 Jun 2015 16:48:28 +0900 Subject: [PATCH 199/217] Fix example greet --- example/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/main.go b/example/main.go index 9ea2b0f..b2bcf6f 100644 --- a/example/main.go +++ b/example/main.go @@ -90,7 +90,7 @@ func Root(w http.ResponseWriter, r *http.Request) { // you to the created greet. // // To post a new greet, try this at a shell: -// $ now=$(date +'%Y-%m-%mT%H:%M:%SZ') +// $ now=$(date +'%Y-%m-%dT%H:%M:%SZ') // $ curl -i -d "user=carl&message=Hello+World&time=$now" localhost:8000/greets func NewGreet(w http.ResponseWriter, r *http.Request) { var greet Greet From 5b91951757614f617847137108b617576047d9b1 Mon Sep 17 00:00:00 2001 From: David Bartley Date: Thu, 18 Jun 2015 22:27:57 -0700 Subject: [PATCH 200/217] Add goji.ServeTLS and goji.ServeListener functions. There's already a ListenAndServeTLS function, but Serve conveniently handles calling various graceful and bind methods. --- serve.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/serve.go b/serve.go index d90e818..a09d1d4 100644 --- a/serve.go +++ b/serve.go @@ -3,8 +3,10 @@ package goji import ( + "crypto/tls" "flag" "log" + "net" "net/http" "time" @@ -22,6 +24,16 @@ func init() { // Serve starts Goji using reasonable defaults. func Serve() { + ServeListener(bind.Default()) +} + +// Like Serve, but enables TLS using the given config. +func ServeTLS(config *tls.Config) { + ServeListener(tls.NewListener(bind.Default(), config)) +} + +// Like Serve, but runs Goji on top of an arbitrary net.Listener. +func ServeListener(listener net.Listener) { if !flag.Parsed() { flag.Parse() } @@ -31,7 +43,6 @@ func Serve() { // This allows packages like expvar to continue working as expected. http.Handle("/", DefaultMux) - listener := bind.Default() log.Println("Starting Goji on", listener.Addr()) graceful.HandleSignals() From 51390a05d196498d546e8a17d55cfbef1b6a0a95 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 30 Jun 2015 14:03:04 -0700 Subject: [PATCH 201/217] Delete test of premature optimization I suspect this is worth some perf, but it's probably not enough for us to care. More importantly, it breaks in Go 1.5, where a recent optimization made sync.WaitGroup much smaller. --- graceful/listener/shard_test.go | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 graceful/listener/shard_test.go diff --git a/graceful/listener/shard_test.go b/graceful/listener/shard_test.go deleted file mode 100644 index 9b99394..0000000 --- a/graceful/listener/shard_test.go +++ /dev/null @@ -1,21 +0,0 @@ -// +build amd64 - -package listener - -import ( - "testing" - "unsafe" -) - -// We pack shards together in an array, but we don't want them packed too -// closely, since we want to give each shard a dedicated CPU cache line. This -// test checks this property for x64 (which has a 64-byte cache line), which -// probably covers the majority of deployments. -// -// As always, this is probably a premature optimization. -func TestShardSize(t *testing.T) { - s := unsafe.Sizeof(shard{}) - if s < 64 { - t.Errorf("sizeof(shard) = %d; expected >64", s) - } -} From 68eff826e9d411ae862edacb57dc80249d23c4b0 Mon Sep 17 00:00:00 2001 From: David Bartley Date: Mon, 29 Jun 2015 23:57:50 -0700 Subject: [PATCH 202/217] Ensure flags are parsed before calling bind.Default(). Tested by running `./example -bind localhost:1234`. --- serve.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/serve.go b/serve.go index a09d1d4..da73a9b 100644 --- a/serve.go +++ b/serve.go @@ -24,20 +24,24 @@ func init() { // Serve starts Goji using reasonable defaults. func Serve() { + if !flag.Parsed() { + flag.Parse() + } + ServeListener(bind.Default()) } // Like Serve, but enables TLS using the given config. func ServeTLS(config *tls.Config) { + if !flag.Parsed() { + flag.Parse() + } + ServeListener(tls.NewListener(bind.Default(), config)) } // Like Serve, but runs Goji on top of an arbitrary net.Listener. func ServeListener(listener net.Listener) { - if !flag.Parsed() { - flag.Parse() - } - DefaultMux.Compile() // Install our handler at the root of the standard net/http default mux. // This allows packages like expvar to continue working as expected. From 831272e8b15c8858656e23ed6065f8202b149851 Mon Sep 17 00:00:00 2001 From: Taichi Sasaki Date: Sat, 25 Jul 2015 08:49:16 +0900 Subject: [PATCH 203/217] Fix a typo --- web/web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/web.go b/web/web.go index 91bbedc..21f6fcc 100644 --- a/web/web.go +++ b/web/web.go @@ -63,7 +63,7 @@ accepted: - types that implement Pattern - string, which is interpreted as a Sinatra-like URL pattern. In particular, the following syntax is recognized: - - a path segment starting with with a colon will match any + - a path segment starting with a colon will match any string placed at that position. e.g., "/:name" will match "/carl", binding "name" to "carl". - a pattern ending with "/*" will match any route with that From 6e46287fcc5439fe8691c56394c311aa077bb3e6 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Mon, 3 Aug 2015 23:26:37 -0700 Subject: [PATCH 204/217] graceful: use TCP keep-alives This is a copy of commit d6bce32a3607222075734bf4363ca3fea02ea1e5 from Go core. --- graceful/server.go | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/graceful/server.go b/graceful/server.go index 4176829..40c1d23 100644 --- a/graceful/server.go +++ b/graceful/server.go @@ -4,10 +4,29 @@ import ( "crypto/tls" "net" "net/http" + "time" ) // Most of the code here is lifted straight from net/http +// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted +// connections. It's used by ListenAndServe and ListenAndServeTLS so +// dead TCP connections (e.g. closing laptop mid-download) eventually +// go away. +type tcpKeepAliveListener struct { + *net.TCPListener +} + +func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { + tc, err := ln.AcceptTCP() + if err != nil { + return + } + tc.SetKeepAlive(true) + tc.SetKeepAlivePeriod(3 * time.Minute) + return tc, nil +} + // A Server is exactly the same as an http.Server, but provides more graceful // implementations of its methods. type Server http.Server @@ -18,11 +37,11 @@ func (srv *Server) ListenAndServe() error { if addr == "" { addr = ":http" } - l, e := net.Listen("tcp", addr) - if e != nil { - return e + ln, err := net.Listen("tcp", addr) + if err != nil { + return err } - return srv.Serve(l) + return srv.Serve(tcpKeepAliveListener{ln.(*net.TCPListener)}) } // ListenAndServeTLS behaves like the method on net/http.Server with the same @@ -52,12 +71,12 @@ func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { return err } - conn, err := net.Listen("tcp", addr) + ln, err := net.Listen("tcp", addr) if err != nil { return err } - tlsListener := tls.NewListener(conn, config) + tlsListener := tls.NewListener(tcpKeepAliveListener{ln.(*net.TCPListener)}, config) return srv.Serve(tlsListener) } From 46b46e00f46dda650e04f0bceeb7c40732cd5988 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Tue, 4 Aug 2015 00:23:56 -0700 Subject: [PATCH 205/217] graceful: opportunistic keep-alives for Serve --- graceful/server.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/graceful/server.go b/graceful/server.go index 40c1d23..8b17295 100644 --- a/graceful/server.go +++ b/graceful/server.go @@ -96,8 +96,13 @@ func ListenAndServeTLS(addr, certfile, keyfile string, handler http.Handler) err return server.ListenAndServeTLS(certfile, keyfile) } -// Serve behaves exactly like the net/http function of the same name. +// Serve mostly behaves like the net/http function of the same name, except that +// if the passed listener is a net.TCPListener, TCP keep-alives are enabled on +// accepted connections. func Serve(l net.Listener, handler http.Handler) error { + if tl, ok := l.(*net.TCPListener); ok { + l = tcpKeepAliveListener{tl} + } server := &Server{Handler: handler} return server.Serve(l) } From be915f805dc15907f50100d9b9dc7ad29e197fb8 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Wed, 11 Nov 2015 13:39:25 -0800 Subject: [PATCH 206/217] pattern: export Raw for retrieving original input This is in addition to the String method, which is oriented more towards human debugging. --- web/regexp_pattern.go | 4 ++++ web/string_pattern.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/web/regexp_pattern.go b/web/regexp_pattern.go index f9f8ecc..95e7e09 100644 --- a/web/regexp_pattern.go +++ b/web/regexp_pattern.go @@ -48,6 +48,10 @@ func (p regexpPattern) String() string { return fmt.Sprintf("regexpPattern(%v)", p.re) } +func (p regexpPattern) Raw() *regexp.Regexp { + return p.re +} + /* I'm sorry, dear reader. I really am. diff --git a/web/string_pattern.go b/web/string_pattern.go index 9e9945a..aa9b33a 100644 --- a/web/string_pattern.go +++ b/web/string_pattern.go @@ -90,6 +90,10 @@ func (s stringPattern) String() string { return fmt.Sprintf("stringPattern(%q)", s.raw) } +func (s stringPattern) Raw() string { + return s.raw +} + // "Break characters" are characters that can end patterns. They are not allowed // to appear in pattern names. "/" was chosen because it is the standard path // separator, and "." was chosen because it often delimits file extensions. ";" From 3bdac6dcc6849f87e5169f2b677fbdb7b31fd09a Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 14 Nov 2015 21:40:42 -0800 Subject: [PATCH 207/217] Fix travis This only breaks in go tip. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 769961d..50f6612 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,4 +23,4 @@ matrix: - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v script: - - go test -v ./... -cover + - go test -cover ./... From 09c4160eb01c64b5a7d3b8dfd93712828143c4ce Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 14 Nov 2015 21:42:53 -0800 Subject: [PATCH 208/217] travis: test 1.5 specifically --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 50f6612..8fe88e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,11 @@ matrix: - go get golang.org/x/tools/cmd/cover - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v + - go: 1.5 + install: + - go get golang.org/x/tools/cmd/cover + - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v + - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v - go: tip install: - go get golang.org/x/tools/cmd/cover From 23a88f8df0ccd37ea84ca3ff36f0faf9edbfc9a2 Mon Sep 17 00:00:00 2001 From: Scott Fleckenstein Date: Fri, 20 Nov 2015 08:29:27 -0800 Subject: [PATCH 209/217] Add `flushWriter` to `mutil.WrapWriter()` This allows httptest.ResponseRecorder to be wrapped and still maintain its Flusher compatibility --- web/mutil/writer_proxy.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/web/mutil/writer_proxy.go b/web/mutil/writer_proxy.go index d572812..9f6d776 100644 --- a/web/mutil/writer_proxy.go +++ b/web/mutil/writer_proxy.go @@ -39,6 +39,9 @@ func WrapWriter(w http.ResponseWriter) WriterProxy { if cn && fl && hj && rf { return &fancyWriter{bw} } + if fl { + return &flushWriter{bw} + } return &bw } @@ -123,3 +126,14 @@ var _ http.CloseNotifier = &fancyWriter{} var _ http.Flusher = &fancyWriter{} var _ http.Hijacker = &fancyWriter{} var _ io.ReaderFrom = &fancyWriter{} + +type flushWriter struct { + basicWriter +} + +func (f *flushWriter) Flush() { + fl := f.basicWriter.ResponseWriter.(http.Flusher) + fl.Flush() +} + +var _ http.Flusher = &flushWriter{} From 47e31837aabfe0f9766fb85c1070fb33e837cfa8 Mon Sep 17 00:00:00 2001 From: Simon Guest Date: Fri, 8 Jan 2016 15:59:32 +1300 Subject: [PATCH 210/217] Added urlquery middleware, resolves #174. --- web/middleware/urlquery.go | 24 +++++++++++++++ web/middleware/urlquery_test.go | 53 +++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 web/middleware/urlquery.go create mode 100644 web/middleware/urlquery_test.go diff --git a/web/middleware/urlquery.go b/web/middleware/urlquery.go new file mode 100644 index 0000000..3738544 --- /dev/null +++ b/web/middleware/urlquery.go @@ -0,0 +1,24 @@ +package middleware + +import ( + "github.com/zenazn/goji/web" + "net/http" +) + +// UrlQueryKey is the context key for the URL Query +const UrlQueryKey string = "urlquery" + +// UrlQuery is a middleware to parse the URL Query parameters just once, +// and store the resulting url.Values in the context. +func UrlQuery(c *web.C, h http.Handler) http.Handler { + fn := func(w http.ResponseWriter, r *http.Request) { + if c.Env == nil { + c.Env = make(map[interface{}]interface{}) + } + c.Env[UrlQueryKey] = r.URL.Query() + + h.ServeHTTP(w, r) + } + + return http.HandlerFunc(fn) +} diff --git a/web/middleware/urlquery_test.go b/web/middleware/urlquery_test.go new file mode 100644 index 0000000..2364191 --- /dev/null +++ b/web/middleware/urlquery_test.go @@ -0,0 +1,53 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "testing" + + "github.com/zenazn/goji/web" +) + +func testUrlQuery(r *http.Request, f func(*web.C, http.ResponseWriter, *http.Request)) *httptest.ResponseRecorder { + var c web.C + + h := func(w http.ResponseWriter, r *http.Request) { + f(&c, w, r) + } + m := UrlQuery(&c, http.HandlerFunc(h)) + w := httptest.NewRecorder() + m.ServeHTTP(w, r) + + return w +} + +func TestUrlQuery(t *testing.T) { + type testcase struct { + url string + expectedParams url.Values + } + + // we're not testing url.Query() here, but rather that the results of the query + // appear in the context + testcases := []testcase{ + testcase{"/", url.Values{}}, + testcase{"/?a=1&b=2&a=3", url.Values{"a": []string{"1", "3"}, "b": []string{"2"}}}, + testcase{"/?x=1&y=2&z=3#freddyishere", url.Values{"x": []string{"1"}, "y": []string{"2"}, "z": []string{"3"}}}, + } + + for _, tc := range testcases { + r, _ := http.NewRequest("GET", tc.url, nil) + testUrlQuery(r, + func(c *web.C, w http.ResponseWriter, r *http.Request) { + params := c.Env[UrlQueryKey].(url.Values) + if !reflect.DeepEqual(params, tc.expectedParams) { + t.Errorf("GET %s, UrlQuery middleware found %v, should be %v", tc.url, params, tc.expectedParams) + } + + w.Write([]byte{'h', 'i'}) + }, + ) + } +} From 0835e6ab6f0a383b027f4453efbebf2c71aa542e Mon Sep 17 00:00:00 2001 From: Simon Guest Date: Sun, 10 Jan 2016 17:45:33 +1300 Subject: [PATCH 211/217] Name change all Url to URL in URLQuery middleware. --- web/middleware/urlquery.go | 10 +++++----- web/middleware/urlquery_test.go | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/web/middleware/urlquery.go b/web/middleware/urlquery.go index 3738544..36c8820 100644 --- a/web/middleware/urlquery.go +++ b/web/middleware/urlquery.go @@ -5,17 +5,17 @@ import ( "net/http" ) -// UrlQueryKey is the context key for the URL Query -const UrlQueryKey string = "urlquery" +// URLQueryKey is the context key for the URL Query +const URLQueryKey string = "urlquery" -// UrlQuery is a middleware to parse the URL Query parameters just once, +// URLQuery is a middleware to parse the URL Query parameters just once, // and store the resulting url.Values in the context. -func UrlQuery(c *web.C, h http.Handler) http.Handler { +func URLQuery(c *web.C, h http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { if c.Env == nil { c.Env = make(map[interface{}]interface{}) } - c.Env[UrlQueryKey] = r.URL.Query() + c.Env[URLQueryKey] = r.URL.Query() h.ServeHTTP(w, r) } diff --git a/web/middleware/urlquery_test.go b/web/middleware/urlquery_test.go index 2364191..d9b7833 100644 --- a/web/middleware/urlquery_test.go +++ b/web/middleware/urlquery_test.go @@ -10,20 +10,20 @@ import ( "github.com/zenazn/goji/web" ) -func testUrlQuery(r *http.Request, f func(*web.C, http.ResponseWriter, *http.Request)) *httptest.ResponseRecorder { +func testURLQuery(r *http.Request, f func(*web.C, http.ResponseWriter, *http.Request)) *httptest.ResponseRecorder { var c web.C h := func(w http.ResponseWriter, r *http.Request) { f(&c, w, r) } - m := UrlQuery(&c, http.HandlerFunc(h)) + m := URLQuery(&c, http.HandlerFunc(h)) w := httptest.NewRecorder() m.ServeHTTP(w, r) return w } -func TestUrlQuery(t *testing.T) { +func TestURLQuery(t *testing.T) { type testcase struct { url string expectedParams url.Values @@ -39,11 +39,11 @@ func TestUrlQuery(t *testing.T) { for _, tc := range testcases { r, _ := http.NewRequest("GET", tc.url, nil) - testUrlQuery(r, + testURLQuery(r, func(c *web.C, w http.ResponseWriter, r *http.Request) { - params := c.Env[UrlQueryKey].(url.Values) + params := c.Env[URLQueryKey].(url.Values) if !reflect.DeepEqual(params, tc.expectedParams) { - t.Errorf("GET %s, UrlQuery middleware found %v, should be %v", tc.url, params, tc.expectedParams) + t.Errorf("GET %s, URLQuery middleware found %v, should be %v", tc.url, params, tc.expectedParams) } w.Write([]byte{'h', 'i'}) From 6b97834061496d40e5b1939f91b0fa5b0c225a53 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 4 Feb 2016 17:37:59 -0800 Subject: [PATCH 212/217] Update code coverage path --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8fe88e4..e07f6e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,12 @@ matrix: include: - go: 1.2 install: - - go get code.google.com/p/go.tools/cmd/cover + - go get golang.org/x/tools/cmd/cover - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v - go: 1.3 install: - - go get code.google.com/p/go.tools/cmd/cover + - go get golang.org/x/tools/cmd/cover - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v - go: 1.4 @@ -19,12 +19,10 @@ matrix: - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v - go: 1.5 install: - - go get golang.org/x/tools/cmd/cover - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v - go: tip install: - - go get golang.org/x/tools/cmd/cover - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v script: From 859bfa93a8db583de1dd72a1edcbf4bb75c0dbbe Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 11 Feb 2016 22:19:35 -0800 Subject: [PATCH 213/217] README: only look at test status for master --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 900d2e0..90d7d17 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Goji ==== -[![GoDoc](https://godoc.org/github.com/zenazn/goji/web?status.svg)](https://godoc.org/github.com/zenazn/goji/web) [![Build Status](https://travis-ci.org/zenazn/goji.svg)](https://travis-ci.org/zenazn/goji) +[![GoDoc](https://godoc.org/github.com/zenazn/goji/web?status.svg)](https://godoc.org/github.com/zenazn/goji/web) [![Build Status](https://travis-ci.org/zenazn/goji.svg?branch=master)](https://travis-ci.org/zenazn/goji) Goji is a minimalistic web framework that values composability and simplicity. From a01dd37b8ff078750dfe49db6031b7a23f14e2d5 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 3 Mar 2016 20:34:44 -0800 Subject: [PATCH 214/217] graceful: import cloneTLSConfig from net/http --- graceful/clone.go | 11 +++++++++++ graceful/clone16.go | 34 ++++++++++++++++++++++++++++++++++ graceful/server.go | 2 +- 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 graceful/clone.go create mode 100644 graceful/clone16.go diff --git a/graceful/clone.go b/graceful/clone.go new file mode 100644 index 0000000..a9027e5 --- /dev/null +++ b/graceful/clone.go @@ -0,0 +1,11 @@ +// +build !go1.6 + +package graceful + +import "crypto/tls" + +// see clone16.go +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + c := *cfg + return &c +} diff --git a/graceful/clone16.go b/graceful/clone16.go new file mode 100644 index 0000000..810b3a2 --- /dev/null +++ b/graceful/clone16.go @@ -0,0 +1,34 @@ +// +build go1.6 + +package graceful + +import "crypto/tls" + +// cloneTLSConfig was taken from the Go standard library's net/http package. We +// need it because tls.Config objects now contain a sync.Once. +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + return &tls.Config{ + Rand: cfg.Rand, + Time: cfg.Time, + Certificates: cfg.Certificates, + NameToCertificate: cfg.NameToCertificate, + GetCertificate: cfg.GetCertificate, + RootCAs: cfg.RootCAs, + NextProtos: cfg.NextProtos, + ServerName: cfg.ServerName, + ClientAuth: cfg.ClientAuth, + ClientCAs: cfg.ClientCAs, + InsecureSkipVerify: cfg.InsecureSkipVerify, + CipherSuites: cfg.CipherSuites, + PreferServerCipherSuites: cfg.PreferServerCipherSuites, + SessionTicketsDisabled: cfg.SessionTicketsDisabled, + SessionTicketKey: cfg.SessionTicketKey, + ClientSessionCache: cfg.ClientSessionCache, + MinVersion: cfg.MinVersion, + MaxVersion: cfg.MaxVersion, + CurvePreferences: cfg.CurvePreferences, + } +} diff --git a/graceful/server.go b/graceful/server.go index 8b17295..ae9a5fb 100644 --- a/graceful/server.go +++ b/graceful/server.go @@ -58,7 +58,7 @@ func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { MinVersion: tls.VersionTLS10, } if srv.TLSConfig != nil { - *config = *srv.TLSConfig + config = cloneTLSConfig(srv.TLSConfig) } if config.NextProtos == nil { config.NextProtos = []string{"http/1.1"} From e9a3f0275121c2439e2bd4b5dc6a9eb3fbe88451 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Thu, 3 Mar 2016 20:43:53 -0800 Subject: [PATCH 215/217] travis: test under Go 1.6 --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index e07f6e7..35410e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,10 @@ matrix: install: - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v + - go: 1.6 + install: + - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v + - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -v - go: tip install: - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -v From a895a84fc1b5af38f6beea5cc2af23e53288bc26 Mon Sep 17 00:00:00 2001 From: Christian Rocha Date: Thu, 17 Mar 2016 00:03:35 -0400 Subject: [PATCH 216/217] =?UTF-8?q?Well,=20actually,=20it=E2=80=99s=202016?= =?UTF-8?q?=20at=20this=20point.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 128f3fc..446aba0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2014, 2015 Carl Jackson (carl@avtok.com) +Copyright (c) 2014, 2015, 2016 Carl Jackson (carl@avtok.com) MIT License From 64eb34159fe53473206c2b3e70fe396a639452f2 Mon Sep 17 00:00:00 2001 From: Carl Jackson Date: Sat, 7 May 2016 13:19:45 -0700 Subject: [PATCH 217/217] Document deprecation, stability This has been a long time coming; apologies for taking so long! Fixes #187, #181, #14 --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 90d7d17..82df6fe 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,15 @@ Goji Goji is a minimalistic web framework that values composability and simplicity. +This project has been superseded by a [new version of Goji][goji2] by the same +author, which has very similar primitives and semantics, but has been updated to +reflect several years of experience with this library and the surrounding Go +ecosystem. This project is still well-loved and well-maintained, and will be for +the foreseeable future, but new projects are encouraged to use `goji.io` +instead. + +[goji2]: https://goji.io + Example ------- @@ -54,6 +63,17 @@ Features [pattern]: https://godoc.org/github.com/zenazn/goji/web#Pattern +Stability +--------- + +Goji's API is essentially frozen, and guarantees to never break compatibility +with existing code (under similar rules to the Go project's +[guidelines][compat]). Goji is suitable for use in production, and has served +billions of requests across several companies. + +[compat]: https://golang.org/doc/go1compat + + Is it any good? ---------------