Creation of Entomologist

A new controller that pins together issues and test targets
Issue kubernetes#10701

prow/github/client.go

@@ -98,6 +98,7 @@ type IssueClient interface {
 	CloseIssue(org, repo string, number int) error
 	ReopenIssue(org, repo string, number int) error
 	FindIssues(query, sort string, asc bool) ([]Issue, error)
+	ListOpenIssues(org, repo string) ([]Issue, error)
 	GetIssue(org, repo string, number int) (*Issue, error)
 	EditIssue(org, repo string, number int, issue *Issue) (*Issue, error)
 }
@@ -1221,6 +1222,34 @@ func (c *client) ListIssueComments(org, repo string, number int) ([]IssueComment
 	return comments, nil
 }
 
+// ListOpenIssues returns all open issues, including pull requests
+//
+// Each page of results consumes one API token.
+//
+// See https://developer.github.com/v3/issues/#list-issues-for-a-repository
+func (c *client) ListOpenIssues(org, repo string) ([]Issue, error) {
+	c.log("ListOpenIssues", org, repo)
+	if c.fake {
+		return nil, nil
+	}
+	path := fmt.Sprintf("/repos/%s/%s/issues", org, repo)
+	var issues []Issue
+	err := c.readPaginatedResults(
+		path,
+		acceptNone,
+		func() interface{} {
+			return &[]Issue{}
+		},
+		func(obj interface{}) {
+			issues = append(issues, *(obj.(*[]Issue))...)
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+	return issues, nil
+}
+
 // GetPullRequests get all open pull requests for a repo.
 //
 // See https://developer.github.com/v3/pulls/#list-pull-requests

prow/github/client_test.go

@@ -414,6 +414,42 @@ func TestCreateStatus(t *testing.T) {
 	}
 }
 
+func TestListIssues(t *testing.T) {
+	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodGet {
+			t.Errorf("Bad method: %s", r.Method)
+		}
+		if r.URL.Path == "/repos/k8s/kuber/issues" {
+			ics := []Issue{{Number: 1}}
+			b, err := json.Marshal(ics)
+			if err != nil {
+				t.Fatalf("Didn't expect error: %v", err)
+			}
+			w.Header().Set("Link", fmt.Sprintf(`<blorp>; rel="first", <https://%s/someotherpath>; rel="next"`, r.Host))
+			fmt.Fprint(w, string(b))
+		} else if r.URL.Path == "/someotherpath" {
+			ics := []Issue{{Number: 2}}
+			b, err := json.Marshal(ics)
+			if err != nil {
+				t.Fatalf("Didn't expect error: %v", err)
+			}
+			fmt.Fprint(w, string(b))
+		} else {
+			t.Errorf("Bad request path: %s", r.URL.Path)
+		}
+	}))
+	defer ts.Close()
+	c := getClient(ts.URL)
+	ics, err := c.ListOpenIssues("k8s", "kuber")
+	if err != nil {
+		t.Errorf("Didn't expect error: %v", err)
+	} else if len(ics) != 2 {
+		t.Errorf("Expected two issues, found %d: %v", len(ics), ics)
+	} else if ics[0].Number != 1 || ics[1].Number != 2 {
+		t.Errorf("Wrong issue IDs: %v", ics)
+	}
+}
+
 func TestListIssueComments(t *testing.T) {
 	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodGet {

prow/github/fakegithub/fakegithub.go

@@ -116,6 +116,16 @@ func (f *FakeClient) IsMember(org, user string) (bool, error) {
 	return false, nil
 }
 
+// ListOpenIssues returns f.issues
+// To mock a mix of issues and pull requests, see github.Issue.PullRequest
+func (f *FakeClient) ListOpenIssues(org, repo string) ([]github.Issue, error) {
+	var issues []github.Issue
+	for _, issue := range f.Issues {
+		issues = append(issues, *issue)
+	}
+	return issues, nil
+}
+
 // ListIssueComments returns comments.
 func (f *FakeClient) ListIssueComments(owner, repo string, number int) ([]github.IssueComment, error) {
 	return append([]github.IssueComment{}, f.IssueComments[number]...), nil

testgrid/BUILD.bazel

@@ -20,6 +20,7 @@ filegroup(
         ":package-srcs",
         "//testgrid/cluster:all-srcs",
         "//testgrid/cmd/configurator:all-srcs",
+        "//testgrid/cmd/entomologist:all-srcs",
         "//testgrid/cmd/updater:all-srcs",
         "//testgrid/config:all-srcs",
         "//testgrid/images:all-srcs",

testgrid/cmd/entomologist/BUILD.bazel

@@ -0,0 +1,50 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "k8s.io/test-infra/testgrid/cmd/entomologist",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//pkg/io:go_default_library",
+        "//prow/config/secret:go_default_library",
+        "//prow/flagutil:go_default_library",
+        "//prow/github:go_default_library",
+        "//testgrid/issue_state:go_default_library",
+        "//vendor/github.com/golang/protobuf/proto:go_default_library",
+        "//vendor/github.com/sirupsen/logrus:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "entomologist",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["main_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//prow/flagutil:go_default_library",
+        "//prow/github:go_default_library",
+        "//prow/github/fakegithub:go_default_library",
+        "//testgrid/issue_state:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

testgrid/cmd/entomologist/README.md

@@ -0,0 +1,34 @@
+# Entomologist
+
+Entomologist collects bugs from GitHub and pins them to TestGrid tests. When an issue is "pinned"
+to a TestGrid target, a hyperlink to the issue will appear on that row.
+
+## Pinning Behavior
+
+Entomologist is expecting targets to be explicitly called out in issues. You can do this by writing
+"target:" at the start of a new line, and then the target you want to pin to.
+
+
+>Some update caused these tests to start failing!
+>
+>target: [sig-storage] ConfigMap binary data should be reflected in volume [NodeConformance] [Conformance] [coreos-beta]
+>
+>target: [sig-storage] ConfigMap should be consumable from pods in volume as non-root [LinuxOnly] [NodeConformance] [Conformance] [coreos-beta]
+>
+>/help
+
+
+GitHub calls Pull Requests "Issues", but Entomologist doesn't.
+Targets in Pull Requests won't be pinned.
+
+Entomologist can be configured with a caching proxy, such as [ghProxy](https://github.com/kubernetes/test-infra/tree/master/ghproxy),
+to minimize API token usage.
+Entomologist is writing an [issue_state.proto](https://github.com/kubernetes/test-infra/blob/master/testgrid/issue_state/issue_state.proto)
+file to Google Cloud Storage (GCS). TestGrid consumes the information placed there.
+
+## Required Flags
+
+- `--github-org/--github-repo`: The organization/repository you're looking through for issues.\
+  For example, this org/repo is `--github-org=kubernetes --github-repo=test-infra`
+- `--output`: The location of the issue_state.proto file that Entomologist will write to
+- `--gcs-credentials-file`: GCS requires credentials to write to a GCS location