Merge pull request #45 from juev/concurency

feat: use concurrent request to github

github.go

@@ -8,13 +8,12 @@ import (
 
 	"github.com/google/go-github/v57/github"
 	"github.com/gregjones/httpcache"
+	"github.com/sourcegraph/conc"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 )
 
 const (
-	// perPage is how many links we get by one shoot
-	perPage int = 100
 	// repositoriesCount is const for allocation memory to store repositories
 	repositoriesCount = 1000
 	// langReposCount is const for allocation memory to store langRepo
@@ -50,15 +49,34 @@ func (g *GitHub) GetRepositories(ctx context.Context) (langRepoMap map[string][]
 	repositories = make([]Repository, 0, repositoriesCount)
 	langRepoMap = make(map[string][]Repository, langReposCount)
 
-	opt := &github.ActivityListStarredOptions{
-		ListOptions: github.ListOptions{PerPage: perPage},
+	_, resp, err := g.client.Activity.ListStarred(ctx, username, nil)
+	if err != nil {
+		log.Fatalln("Error: cannot fetch starred:", err)
 	}
 
-	for {
-		repos, resp, err := g.client.Activity.ListStarred(ctx, username, opt)
-		if err != nil {
-			log.Fatalln("Error: cannot fetch starred:", err)
+	ch := make(chan []*github.StarredRepository, 1)
+	go func() {
+		wg := conc.WaitGroup{}
+		for i := 1; i <= resp.LastPage; i++ {
+			i := i
+			wg.Go(func() {
+				opt := &github.ActivityListStarredOptions{
+					ListOptions: github.ListOptions{
+						Page: i,
+					},
+				}
+				repos, _, err := g.client.Activity.ListStarred(ctx, username, opt)
+				if err != nil {
+					log.Fatalln("Error: cannot fetch starred:", err)
+				}
+				ch <- repos
+			})
 		}
+		wg.Wait()
+		close(ch)
+	}()
+
+	for repos := range ch {
 		for _, r := range repos {
 			repo := Repository{
 				FullName:    r.Repository.GetFullName(),
@@ -77,11 +95,6 @@ func (g *GitHub) GetRepositories(ctx context.Context) (langRepoMap map[string][]
 			}
 			langRepoMap[lang] = append(langRepoMap[lang], repo)
 		}
-
-		if resp.NextPage == 0 {
-			break
-		}
-		opt.Page = resp.NextPage
 	}
 
 	if len(repositories) == 0 {

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/google/go-github/v57 v57.0.0
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
+	github.com/sourcegraph/conc v0.3.0
 	github.com/spf13/pflag v1.0.5
 	golang.org/x/text v0.14.0
 )

go.sum

@@ -1,3 +1,4 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-github/v57 v57.0.0 h1:L+Y3UPTY8ALM8x+TV0lg+IEBI+upibemtBD8Q9u7zHs=
@@ -6,8 +7,13 @@ github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=