forked from influxdata/telegraf
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add github input plugin (influxdata#5587)
- Loading branch information
Showing
6 changed files
with
372 additions
and
0 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
# GitHub Input Plugin | ||
|
||
The [GitHub](https://www.github.com) input plugin gathers statistics from GitHub repositories. | ||
|
||
### Configuration: | ||
|
||
```toml | ||
[[inputs.github]] | ||
## List of repositories to monitor | ||
## ex: repositories = ["influxdata/telegraf"] | ||
# repositories = [] | ||
|
||
## Optional: Unauthenticated requests are limited to 60 per hour. | ||
# access_token = "" | ||
|
||
## Optional: Default 5s. | ||
# http_timeout = "5s" | ||
``` | ||
|
||
### Metrics: | ||
|
||
- github_repository | ||
- tags: | ||
- `name` - The repository name | ||
- `owner` - The owner of the repository | ||
- `language` - The primary language of the repository | ||
- `license` - The license set for the repository | ||
- fields: | ||
- `stars` (int) | ||
- `forks` (int) | ||
- `open_issues` (int) | ||
- `size` (int) | ||
|
||
* github_rate_limit | ||
- tags: | ||
- `access_token` - An obfusticated reference to the configured access token or "Unauthenticated" | ||
- fields: | ||
- `limit` - How many requests you are limited to (per hour) | ||
- `remaining` - How many requests you have remaining (per hour) | ||
- `blocks` - How many requests have been blocked due to rate limit | ||
|
||
### Example Output: | ||
|
||
``` | ||
github,full_name=influxdata/telegraf,name=telegraf,owner=influxdata,language=Go,license=MIT\ License stars=6401i,forks=2421i,open_issues=722i,size=22611i 1552651811000000000 | ||
internal_github,access_token=Unauthenticated rate_limit_remaining=59i,rate_limit_limit=60i,rate_limit_blocks=0i 1552653551000000000 | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,184 @@ | ||
package github | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"net/http" | ||
"strings" | ||
"sync" | ||
"time" | ||
|
||
"github.com/google/go-github/github" | ||
"github.com/influxdata/telegraf" | ||
"github.com/influxdata/telegraf/internal" | ||
"github.com/influxdata/telegraf/plugins/inputs" | ||
"github.com/influxdata/telegraf/selfstat" | ||
"golang.org/x/oauth2" | ||
) | ||
|
||
// GitHub - plugin main structure | ||
type GitHub struct { | ||
Repositories []string `toml:"repositories"` | ||
AccessToken string `toml:"access_token"` | ||
HTTPTimeout internal.Duration `toml:"http_timeout"` | ||
githubClient *github.Client | ||
|
||
obfusticatedToken string | ||
|
||
RateLimit selfstat.Stat | ||
RateLimitErrors selfstat.Stat | ||
RateRemaining selfstat.Stat | ||
} | ||
|
||
const sampleConfig = ` | ||
## List of repositories to monitor | ||
## ex: repositories = ["influxdata/telegraf"] | ||
# repositories = [] | ||
## Optional: Unauthenticated requests are limited to 60 per hour. | ||
# access_token = "" | ||
## Optional: Default 5s. | ||
# http_timeout = "5s" | ||
` | ||
|
||
// SampleConfig returns sample configuration for this plugin. | ||
func (g *GitHub) SampleConfig() string { | ||
return sampleConfig | ||
} | ||
|
||
// Description returns the plugin description. | ||
func (g *GitHub) Description() string { | ||
return "Read repository information from GitHub, including forks, stars, and more." | ||
} | ||
|
||
// Create GitHub Client | ||
func (g *GitHub) createGitHubClient(ctx context.Context) (*github.Client, error) { | ||
httpClient := &http.Client{ | ||
Transport: &http.Transport{ | ||
Proxy: http.ProxyFromEnvironment, | ||
}, | ||
Timeout: g.HTTPTimeout.Duration, | ||
} | ||
|
||
g.obfusticatedToken = "Unauthenticated" | ||
|
||
if g.AccessToken != "" { | ||
tokenSource := oauth2.StaticTokenSource( | ||
&oauth2.Token{AccessToken: g.AccessToken}, | ||
) | ||
oauthClient := oauth2.NewClient(ctx, tokenSource) | ||
ctx = context.WithValue(ctx, oauth2.HTTPClient, oauthClient) | ||
|
||
g.obfusticatedToken = g.AccessToken[0:4] + "..." + g.AccessToken[len(g.AccessToken)-3:] | ||
|
||
return github.NewClient(oauthClient), nil | ||
} | ||
|
||
return github.NewClient(httpClient), nil | ||
} | ||
|
||
// Gather GitHub Metrics | ||
func (g *GitHub) Gather(acc telegraf.Accumulator) error { | ||
ctx := context.Background() | ||
|
||
if g.githubClient == nil { | ||
githubClient, err := g.createGitHubClient(ctx) | ||
|
||
if err != nil { | ||
return err | ||
} | ||
|
||
g.githubClient = githubClient | ||
|
||
tokenTags := map[string]string{ | ||
"access_token": g.obfusticatedToken, | ||
} | ||
|
||
g.RateLimitErrors = selfstat.Register("github", "rate_limit_blocks", tokenTags) | ||
g.RateLimit = selfstat.Register("github", "rate_limit_limit", tokenTags) | ||
g.RateRemaining = selfstat.Register("github", "rate_limit_remaining", tokenTags) | ||
} | ||
|
||
var wg sync.WaitGroup | ||
wg.Add(len(g.Repositories)) | ||
|
||
for _, repository := range g.Repositories { | ||
go func(repositoryName string, acc telegraf.Accumulator) { | ||
defer wg.Done() | ||
|
||
owner, repository, err := splitRepositoryName(repositoryName) | ||
if err != nil { | ||
acc.AddError(err) | ||
return | ||
} | ||
|
||
repositoryInfo, response, err := g.githubClient.Repositories.Get(ctx, owner, repository) | ||
|
||
if _, ok := err.(*github.RateLimitError); ok { | ||
g.RateLimitErrors.Incr(1) | ||
} | ||
|
||
if err != nil { | ||
acc.AddError(err) | ||
return | ||
} | ||
|
||
g.RateLimit.Set(int64(response.Rate.Limit)) | ||
g.RateRemaining.Set(int64(response.Rate.Remaining)) | ||
|
||
now := time.Now() | ||
tags := getTags(repositoryInfo) | ||
fields := getFields(repositoryInfo) | ||
|
||
acc.AddFields("github_repository", fields, tags, now) | ||
}(repository, acc) | ||
} | ||
|
||
wg.Wait() | ||
return nil | ||
} | ||
|
||
func splitRepositoryName(repositoryName string) (string, string, error) { | ||
splits := strings.SplitN(repositoryName, "/", 2) | ||
|
||
if len(splits) != 2 { | ||
return "", "", fmt.Errorf("%v is not of format 'owner/repository'", repositoryName) | ||
} | ||
|
||
return splits[0], splits[1], nil | ||
} | ||
|
||
func getLicense(repositoryInfo *github.Repository) string { | ||
if repositoryInfo.GetLicense() != nil { | ||
return *repositoryInfo.License.Name | ||
} | ||
|
||
return "None" | ||
} | ||
|
||
func getTags(repositoryInfo *github.Repository) map[string]string { | ||
return map[string]string{ | ||
"owner": *repositoryInfo.Owner.Login, | ||
"name": *repositoryInfo.Name, | ||
"language": *repositoryInfo.Language, | ||
"license": getLicense(repositoryInfo), | ||
} | ||
} | ||
|
||
func getFields(repositoryInfo *github.Repository) map[string]interface{} { | ||
return map[string]interface{}{ | ||
"stars": *repositoryInfo.StargazersCount, | ||
"forks": *repositoryInfo.ForksCount, | ||
"open_issues": *repositoryInfo.OpenIssuesCount, | ||
"size": *repositoryInfo.Size, | ||
} | ||
} | ||
|
||
func init() { | ||
inputs.Add("github", func() telegraf.Input { | ||
return &GitHub{ | ||
HTTPTimeout: internal.Duration{Duration: time.Second * 5}, | ||
} | ||
}) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,119 @@ | ||
package github | ||
|
||
import ( | ||
"reflect" | ||
"testing" | ||
|
||
gh "github.com/google/go-github/github" | ||
"github.com/stretchr/testify/require" | ||
) | ||
|
||
func TestSplitRepositoryNameWithWorkingExample(t *testing.T) { | ||
var validRepositoryNames = []struct { | ||
fullName string | ||
owner string | ||
repository string | ||
}{ | ||
{"influxdata/telegraf", "influxdata", "telegraf"}, | ||
{"influxdata/influxdb", "influxdata", "influxdb"}, | ||
{"rawkode/saltstack-dotfiles", "rawkode", "saltstack-dotfiles"}, | ||
} | ||
|
||
for _, tt := range validRepositoryNames { | ||
t.Run(tt.fullName, func(t *testing.T) { | ||
owner, repository, _ := splitRepositoryName(tt.fullName) | ||
|
||
require.Equal(t, tt.owner, owner) | ||
require.Equal(t, tt.repository, repository) | ||
}) | ||
} | ||
} | ||
|
||
func TestSplitRepositoryNameWithNoSlash(t *testing.T) { | ||
var invalidRepositoryNames = []string{ | ||
"influxdata-influxdb", | ||
} | ||
|
||
for _, tt := range invalidRepositoryNames { | ||
t.Run(tt, func(t *testing.T) { | ||
_, _, err := splitRepositoryName(tt) | ||
|
||
require.NotNil(t, err) | ||
}) | ||
} | ||
} | ||
|
||
func TestGetLicenseWhenExists(t *testing.T) { | ||
licenseName := "MIT" | ||
license := gh.License{Name: &licenseName} | ||
repository := gh.Repository{License: &license} | ||
|
||
getLicenseReturn := getLicense(&repository) | ||
|
||
require.Equal(t, "MIT", getLicenseReturn) | ||
} | ||
|
||
func TestGetLicenseWhenMissing(t *testing.T) { | ||
repository := gh.Repository{} | ||
|
||
getLicenseReturn := getLicense(&repository) | ||
|
||
require.Equal(t, "None", getLicenseReturn) | ||
} | ||
|
||
func TestGetTags(t *testing.T) { | ||
licenseName := "MIT" | ||
license := gh.License{Name: &licenseName} | ||
|
||
ownerName := "influxdata" | ||
owner := gh.User{Login: &ownerName} | ||
|
||
fullName := "influxdata/influxdb" | ||
repositoryName := "influxdb" | ||
|
||
language := "Go" | ||
|
||
repository := gh.Repository{ | ||
FullName: &fullName, | ||
Name: &repositoryName, | ||
License: &license, | ||
Owner: &owner, | ||
Language: &language, | ||
} | ||
|
||
getTagsReturn := getTags(&repository) | ||
|
||
correctTagsReturn := map[string]string{ | ||
"owner": ownerName, | ||
"name": repositoryName, | ||
"language": language, | ||
"license": licenseName, | ||
} | ||
|
||
require.Equal(t, true, reflect.DeepEqual(getTagsReturn, correctTagsReturn)) | ||
} | ||
|
||
func TestGetFields(t *testing.T) { | ||
stars := 1 | ||
forks := 2 | ||
openIssues := 3 | ||
size := 4 | ||
|
||
repository := gh.Repository{ | ||
StargazersCount: &stars, | ||
ForksCount: &forks, | ||
OpenIssuesCount: &openIssues, | ||
Size: &size, | ||
} | ||
|
||
getFieldsReturn := getFields(&repository) | ||
|
||
correctFieldReturn := make(map[string]interface{}) | ||
|
||
correctFieldReturn["stars"] = 1 | ||
correctFieldReturn["forks"] = 2 | ||
correctFieldReturn["open_issues"] = 3 | ||
correctFieldReturn["size"] = 4 | ||
|
||
require.Equal(t, true, reflect.DeepEqual(getFieldsReturn, correctFieldReturn)) | ||
} |