Add table github_search_repository. Closes #104 (#114)

docs/tables/github_search_repository.md

@@ -0,0 +1,75 @@
+# Table: github_search_repository
+
+The `github_search_repository` table helps to find repositories via various criteria. You can search for repositories on GitHub and narrow the results using these repository search qualifiers in any combination.
+
+ **You must always include at least one search term when searching source code** in the where or join clause using the `query` column. You can search for repositories globally across all of GitHub.com, or search for repositories within a particular organization.
+
+## Examples
+
+### Get a specific repository
+
+```sql
+select
+  name,
+  owner_login,
+  language,
+  forks_count,
+  stargazers_count,
+  subscribers_count,
+  watchers_count
+from
+  github_search_repository
+where
+  query = 'repo:turbot/steampipe-plugin-github';
+```
+
+### List repositories based on contents of a repository
+
+```sql
+select
+  name,
+  owner_login,
+  language,
+  forks_count,
+  stargazers_count,
+  subscribers_count,
+  watchers_count
+from
+  github_search_repository
+where
+  query = 'stargazers in:readme repo:turbot/steampipe-plugin-github';
+```
+
+### List repositories with more than 100000 followers
+
+```sql
+select
+  name,
+  owner_login,
+  language,
+  forks_count,
+  stargazers_count,
+  subscribers_count,
+  watchers_count
+from
+  github_search_repository
+where
+  query = 'followers:>=100000';
+```
+
+### List forked repositories created within specific timestamp
+
+```sql
+select
+  name,
+  owner_login,
+  language,
+  forks_count,
+  stargazers_count,
+  subscribers_count,
+  watchers_count
+from
+  github_search_repository
+where
+  query = 'tinyspotifyr in:name created:2021-01-01..2021-01-05 fork:only';
+```

github/plugin.go

@@ -40,6 +40,7 @@ func Plugin(ctx context.Context) *plugin.Plugin {
 			"github_search_commit":       tableGitHubSearchCommit(ctx),
 			"github_search_label":        tableGitHubSearchLable(ctx),
 			"github_search_pull_request": tableGitHubSearchPullRequest(ctx),
+			"github_search_repository":   tableGitHubSearchRepository(ctx),
 			"github_search_topic":        tableGitHubSearchTopic(ctx),
 			"github_search_user":         tableGitHubSearchUser(ctx),
 			"github_stargazer":           tableGitHubStargazer(ctx),

github/table_github_search_repository.go

@@ -0,0 +1,105 @@
+package github
+
+import (
+	"context"
+
+	"github.com/google/go-github/v33/github"
+	"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
+	"github.com/turbot/steampipe-plugin-sdk/plugin"
+	"github.com/turbot/steampipe-plugin-sdk/plugin/transform"
+)
+
+//// TABLE DEFINITION
+
+func tableGitHubSearchRepository(ctx context.Context) *plugin.Table {
+	return &plugin.Table{
+		Name:        "github_search_repository",
+		Description: "Find repositories via various criteria.",
+		List: &plugin.ListConfig{
+			KeyColumns: plugin.SingleColumn("query"),
+			Hydrate:    tableGitHubSearchRepositoryList,
+		},
+		Columns: gitHubSearchRepositoryColumns([]*plugin.Column{
+			{Name: "query", Type: proto.ColumnType_STRING, Transform: transform.FromQual("query"), Description: "The query used to match the repository."},
+			{Name: "text_matches", Type: proto.ColumnType_JSON, Description: "The text match details."},
+		}),
+	}
+}
+
+//// LIST FUNCTION
+
+func tableGitHubSearchRepositoryList(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
+	logger := plugin.Logger(ctx)
+	logger.Trace("tableGitHubSearchRepositoryList")
+
+	quals := d.KeyColumnQuals
+	query := quals["query"].GetStringValue()
+
+	if query == "" {
+		return nil, nil
+	}
+
+	opt := &github.SearchOptions{
+		ListOptions: github.ListOptions{PerPage: 100},
+		TextMatch:   true,
+	}
+
+	type ListPageResponse struct {
+		result *github.RepositoriesSearchResult
+		resp   *github.Response
+	}
+
+	client := connect(ctx, d)
+
+	// Reduce the basic request limit down if the user has only requested a small number of rows
+	limit := d.QueryContext.Limit
+	if limit != nil {
+		if *limit < int64(opt.ListOptions.PerPage) {
+			opt.ListOptions.PerPage = int(*limit)
+		}
+	}
+
+	listPage := func(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
+		result, resp, err := client.Search.Repositories(ctx, query, opt)
+
+		if err != nil {
+			logger.Error("tableGitHubSearchRepositoryList", "error_Search.Repositories", err)
+			return nil, err
+		}
+
+		return ListPageResponse{
+			result: result,
+			resp:   resp,
+		}, nil
+	}
+
+	for {
+		listPageResponse, err := plugin.RetryHydrate(ctx, d, h, listPage, &plugin.RetryConfig{ShouldRetryError: shouldRetryError})
+
+		if err != nil {
+			logger.Error("tableGitHubSearchRepositoryList", "error_RetryHydrate", err)
+			return nil, err
+		}
+
+		listResponse := listPageResponse.(ListPageResponse)
+		repoResults := listResponse.result.Repositories
+		resp := listResponse.resp
+
+		for _, i := range repoResults {
+			d.StreamListItem(ctx, i)
+
+			// Context can be cancelled due to manual cancellation or the limit has been hit
+			if d.QueryStatus.RowsRemaining(ctx) == 0 {
+				return nil, nil
+			}
+		}
+
+		if resp.NextPage == 0 {
+			break
+		}
+
+		opt.Page = resp.NextPage
+	}
+
+	return nil, nil
+}

github/utils.go

@@ -86,3 +86,7 @@ func filterUserLogins(_ context.Context, input *transform.TransformData) (interf
 	}
 	return user_logins, nil
 }
+
+func gitHubSearchRepositoryColumns(columns []*plugin.Column) []*plugin.Column {
+	return append(gitHubRepositoryColumns(), columns...)
+}