From 3aaa6f118996781361f2fa6552b90e31bc953f1f Mon Sep 17 00:00:00 2001 From: Will Norris Date: Wed, 9 Apr 2014 12:40:49 -0700 Subject: [PATCH] first pass at tool for testing JSON fields mapping This tool help to identify fields being returned by the GitHub API that are not currently being mapped onto our Go data types. Some of these may be deliberate omissions, so these checks should only be treated as an aid in identifying these fields, not as an absolute list of fields to be added. --- tests/README.md | 20 +++++ tests/fields/fields.go | 140 +++++++++++++++++++++++++++++++ tests/integration/github_test.go | 3 +- 3 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 tests/fields/fields.go diff --git a/tests/README.md b/tests/README.md index 7f9defd..b059dcd 100644 --- a/tests/README.md +++ b/tests/README.md @@ -30,3 +30,23 @@ be run using a dedicated test account. Run tests using: GITHUB_AUTH_TOKEN=XXX go test -v ./integration + + +fields +------ + +This will identify the fields being returned by the live GitHub API that are +not currently being mapped into the relevant Go data type. Sometimes fields +are deliberately not mapped, so the results of this tool should just be taken +as a hint. + +This test sends real network traffic to the GitHub API and will exhaust the +default unregistered rate limit (60 requests per hour) very quickly. +Additionally, some data is only returned for authenticated API calls. Unlike +the integration tests above, these tests only read data, so it's less +imperitive that these be run using a dedicated test account (though you still +really should). + +Run the fields tool using: + + GITHUB_AUTH_TOKEN=XXX go run ./fields/fields.go diff --git a/tests/fields/fields.go b/tests/fields/fields.go new file mode 100644 index 0000000..8a0eb34 --- /dev/null +++ b/tests/fields/fields.go @@ -0,0 +1,140 @@ +// Copyright 2014 The go-github AUTHORS. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This tool tests for the JSON mappings in the go-github data types. It will +// identify fields that are returned by the live GitHub API, but that are not +// currently mapped into a struct field of the relevant go-github type. This +// helps to ensure that all relevant data returned by the API is being made +// accessible, particularly new fields that are periodically (and sometimes +// quietly) added to the API over time. +// +// These tests simply aid in identifying which fields aren't being mapped; it +// is not necessarily true that every one of them should always be mapped. +// Some fields may be undocumented for a reason, either because they aren't +// actually used yet or should not be relied upon. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "reflect" + "strings" + "code.google.com/p/goauth2/oauth" + + "github.com/google/go-github/github" +) + +var ( + client *github.Client + + // auth indicates whether tests are being run with an OAuth token. + // Tests can use this flag to skip certain tests when run without auth. + auth bool + + skipURLs = flag.Bool("skip_urls", false, "skip url fields") +) + +func main() { + flag.Parse() + + token := os.Getenv("GITHUB_AUTH_TOKEN") + if token == "" { + println("!!! No OAuth token. Some tests won't run. !!!\n") + client = github.NewClient(nil) + } else { + t := &oauth.Transport{ + Token: &oauth.Token{AccessToken: token}, + } + client = github.NewClient(t.Client()) + auth = true + } + + testType("rate_limit", github.Rate{}) + testType("users/octocat", github.User{}) + testType("orgs/google", github.Organization{}) + testType("repos/google/go-github", github.Repository{}) +} + +func checkAuth(name string) bool { + if !auth { + fmt.Printf("No auth - skipping portions of %v\n", name) + } + return auth +} + +// testType fetches the JSON resource at urlStr and compares its keys to the +// struct fields of typ. +// +// TODO: handle resources that are more easily fetched as an array of objects, +// rather than a single object (e.g. a user's public keys). In this case, we +// should just take the first object in the array, and use that. In that case, +// should typ also be specified as a slice? +func testType(urlStr string, typ interface{}) error { + req, err := client.NewRequest("GET", urlStr, nil) + if err != nil { + return err + } + + // I'm thinking we might want to unmarshall the response both as a + // map[string]interface{} as well as typ, though I'm not 100% sure. + // That's why we unmarshal to json.RawMessage first here. + raw := new(json.RawMessage) + _, err = client.Do(req, raw) + if err != nil { + return err + } + + var m map[string]interface{} + err = json.Unmarshal(*raw, &m) + if err != nil { + return err + } + + fields := jsonFields(typ) + + for k, v := range m { + if *skipURLs && strings.HasSuffix(k, "_url") { + continue + } + if _, ok := fields[k]; !ok { + fmt.Printf("%v missing field for key: %v (example value: %v)\n", reflect.TypeOf(typ), k, v) + } + } + + return nil +} + +// parseTag splits a struct field's url tag into its name and comma-separated +// options. +func parseTag(tag string) (string, []string) { + s := strings.Split(tag, ",") + return s[0], s[1:] +} + +// jsonFields returns a map of JSON fields that have an explicit mapping to a +// field in v. The fields will be in the returned map's keys; the map values +// for those keys is currently undefined. +func jsonFields(v interface{}) map[string]interface{} { + fields := make(map[string]interface{}) + + typ := reflect.TypeOf(v) + for i := 0; i < typ.NumField(); i++ { + sf := typ.Field(i) + if sf.PkgPath != "" { // unexported + continue + } + + tag := sf.Tag.Get("json") + if tag == "-" { + continue + } + + name, opts := parseTag(tag) + fields[name] = opts + } + return fields +} diff --git a/tests/integration/github_test.go b/tests/integration/github_test.go index 8110c11..1ac6a0b 100644 --- a/tests/integration/github_test.go +++ b/tests/integration/github_test.go @@ -3,11 +3,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package tests contains integration tests for the go-github library. -// // These tests call the live GitHub API, and therefore require a little more // setup to run. See https://github.com/google/go-github/tree/master/tests/integration // for more information + package tests import (