Add migrate from OneDev (#16356)

* Use context to simplify logic.

* Added migration from OneDev.
This PR adds [OneDev](https://code.onedev.io/) as migration source.

Supported:
- [x] Milestones
- [x] Issues
- [x] Pull Requests
- [x] Comments
- [x] Reviews
- [x] Labels

modules/convert/utils.go

@@ -33,6 +33,8 @@ func ToGitServiceType(value string) structs.GitServiceType {
 		return structs.GitlabService
 	case "gogs":
 		return structs.GogsService
+	case "onedev":
+		return structs.OneDevService
 	default:
 		return structs.PlainGitService
 	}

modules/migrations/base/downloader.go

@@ -13,9 +13,9 @@ import (
 
 // GetCommentOptions represents an options for get comment
 type GetCommentOptions struct {
-	IssueNumber int64
-	Page        int
-	PageSize    int
+	Context  IssueContext
+	Page     int
+	PageSize int
 }
 
 // Downloader downloads the site repo information
@@ -30,7 +30,7 @@ type Downloader interface {
 	GetComments(opts GetCommentOptions) ([]*Comment, bool, error)
 	SupportGetRepoComments() bool
 	GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
-	GetReviews(pullRequestNumber int64) ([]*Review, error)
+	GetReviews(pullRequestContext IssueContext) ([]*Review, error)
 	FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
 }
 

modules/migrations/base/issue.go

@@ -7,6 +7,25 @@ package base
 
 import "time"
 
+// IssueContext is used to map between local and foreign issue/PR ids.
+type IssueContext interface {
+	LocalID() int64
+	ForeignID() int64
+}
+
+// BasicIssueContext is a 1:1 mapping between local and foreign ids.
+type BasicIssueContext int64
+
+// LocalID gets the local id.
+func (c BasicIssueContext) LocalID() int64 {
+	return int64(c)
+}
+
+// ForeignID gets the foreign id.
+func (c BasicIssueContext) ForeignID() int64 {
+	return int64(c)
+}
+
 // Issue is a standard issue information
 type Issue struct {
 	Number      int64
@@ -25,4 +44,5 @@ type Issue struct {
 	Labels      []*Label
 	Reactions   []*Reaction
 	Assignees   []string
+	Context     IssueContext `yaml:"-"`
 }

modules/migrations/base/null_downloader.go

@@ -50,7 +50,7 @@ func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
 	return nil, false, &ErrNotSupported{Entity: "Issues"}
 }
 
-// GetComments returns comments according issueNumber
+// GetComments returns comments according the options
 func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) {
 	return nil, false, &ErrNotSupported{Entity: "Comments"}
 }
@@ -61,7 +61,7 @@ func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool
 }
 
 // GetReviews returns pull requests review
-func (n NullDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
+func (n NullDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
 	return nil, &ErrNotSupported{Entity: "Reviews"}
 }
 

modules/migrations/base/pullrequest.go

@@ -13,7 +13,6 @@ import (
 // PullRequest defines a standard pull request information
 type PullRequest struct {
 	Number         int64
-	OriginalNumber int64 `yaml:"original_number"`
 	Title          string
 	PosterName     string `yaml:"poster_name"`
 	PosterID       int64  `yaml:"poster_id"`
@@ -34,6 +33,7 @@ type PullRequest struct {
 	Assignees      []string
 	IsLocked       bool `yaml:"is_locked"`
 	Reactions      []*Reaction
+	Context        IssueContext `yaml:"-"`
 }
 
 // IsForkPullRequest returns true if the pull request from a forked repository but not the same repository

modules/migrations/base/retry_downloader.go

@@ -182,14 +182,14 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo
 }
 
 // GetReviews returns pull requests reviews
-func (d *RetryDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
+func (d *RetryDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
 	var (
 		reviews []*Review
 		err     error
 	)
 
 	err = d.retry(func() error {
-		reviews, err = d.Downloader.GetReviews(pullRequestNumber)
+		reviews, err = d.Downloader.GetReviews(pullRequestContext)
 		return err
 	})
 

modules/migrations/gitea_downloader.go

@@ -444,6 +444,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err
 			Labels:      labels,
 			Assignees:   assignees,
 			IsLocked:    issue.IsLocked,
+			Context:     base.BasicIssueContext(issue.Index),
 		})
 	}
 
@@ -466,26 +467,26 @@ func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comm
 	default:
 	}
 
-	comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.IssueNumber, gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{
+	comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{
 		// PageSize: g.maxPerPage,
 		// Page:     i,
 	}})
 	if err != nil {
-		return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.IssueNumber, err)
+		return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.Context.ForeignID(), err)
 	}
 
 	for _, comment := range comments {
 		reactions, err := g.getCommentReactions(comment.ID)
 		if err != nil {
-			log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err)
+			log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)
 			if err2 := models.CreateRepositoryNotice(
-				fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err)); err2 != nil {
+				fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil {
 				log.Error("create repository notice failed: ", err2)
 			}
 		}
 
 		allComments = append(allComments, &base.Comment{
-			IssueIndex:  opts.IssueNumber,
+			IssueIndex:  opts.Context.LocalID(),
 			PosterID:    comment.Poster.ID,
 			PosterName:  comment.Poster.UserName,
 			PosterEmail: comment.Poster.Email,
@@ -615,6 +616,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
 				RepoName:  g.repoName,
 				OwnerName: g.repoOwner,
 			},
+			Context: base.BasicIssueContext(pr.Index),
 		})
 	}
 
@@ -626,7 +628,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
 }
 
 // GetReviews returns pull requests review
-func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {
+func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
 	if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil {
 		log.Info("GiteaDownloader: instance to old, skip GetReviews")
 		return nil, nil
@@ -642,7 +644,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {
 		default:
 		}
 
-		prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, index, gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{
+		prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, context.ForeignID(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{
 			Page:     i,
 			PageSize: g.maxPerPage,
 		}})
@@ -652,7 +654,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {
 
 		for _, pr := range prl {
 
-			rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, index, pr.ID)
+			rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, context.ForeignID(), pr.ID)
 			if err != nil {
 				return nil, err
 			}
@@ -678,7 +680,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {
 
 			allReviews = append(allReviews, &base.Review{
 				ID:           pr.ID,
-				IssueIndex:   index,
+				IssueIndex:   context.LocalID(),
 				ReviewerID:   pr.Reviewer.ID,
 				ReviewerName: pr.Reviewer.UserName,
 				Official:     pr.Official,

modules/migrations/gitea_downloader_test.go

@@ -199,7 +199,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
 	}, issues)
 
 	comments, _, err := downloader.GetComments(base.GetCommentOptions{
-		IssueNumber: 4,
+		Context: base.BasicIssueContext(4),
 	})
 	assert.NoError(t, err)
 	assertCommentsEqual(t, []*base.Comment{
@@ -265,7 +265,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
 		PatchURL:       "https://gitea.com/gitea/test_repo/pulls/12.patch",
 	}, prs[1])
 
-	reviews, err := downloader.GetReviews(7)
+	reviews, err := downloader.GetReviews(base.BasicIssueContext(7))
 	assert.NoError(t, err)
 	assertReviewsEqual(t, []*base.Review{
 		{

modules/migrations/gitea_uploader.go

@@ -609,6 +609,9 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 
 	// download patch file
 	err := func() error {
+		if pr.PatchURL == "" {
+			return nil
+		}
 		// pr.PatchURL maybe a local file
 		ret, err := uri.Open(pr.PatchURL)
 		if err != nil {

modules/migrations/github.go

@@ -428,6 +428,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
 			Closed:      issue.ClosedAt,
 			IsLocked:    issue.GetLocked(),
 			Assignees:   assignees,
+			Context:     base.BasicIssueContext(*issue.Number),
 		})
 	}
 
@@ -441,15 +442,15 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool {
 
 // GetComments returns comments according issueNumber
 func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
-	if opts.IssueNumber > 0 {
-		comments, err := g.getComments(opts.IssueNumber)
+	if opts.Context != nil {
+		comments, err := g.getComments(opts.Context)
 		return comments, false, err
 	}
 
 	return g.GetAllComments(opts.Page, opts.PageSize)
 }
 
-func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, error) {
+func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*base.Comment, error) {
 	var (
 		allComments = make([]*base.Comment, 0, g.maxPerPage)
 		created     = "created"
@@ -464,7 +465,7 @@ func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, er
 	}
 	for {
 		g.sleep()
-		comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt)
+		comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt)
 		if err != nil {
 			return nil, fmt.Errorf("error while listing repos: %v", err)
 		}
@@ -495,7 +496,7 @@ func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, er
 			}
 
 			allComments = append(allComments, &base.Comment{
-				IssueIndex:  issueNumber,
+				IssueIndex:  issueContext.LocalID(),
 				PosterID:    comment.GetUser().GetID(),
 				PosterName:  comment.GetUser().GetLogin(),
 				PosterEmail: comment.GetUser().GetEmail(),
@@ -661,6 +662,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
 			},
 			PatchURL:  pr.GetPatchURL(),
 			Reactions: reactions,
+			Context:   base.BasicIssueContext(*pr.Number),
 		})
 	}
 
@@ -724,28 +726,28 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
 }
 
 // GetReviews returns pull requests review
-func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
+func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Review, error) {
 	var allReviews = make([]*base.Review, 0, g.maxPerPage)
 	opt := &github.ListOptions{
 		PerPage: g.maxPerPage,
 	}
 	for {
 		g.sleep()
-		reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), opt)
+		reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt)
 		if err != nil {
 			return nil, fmt.Errorf("error while listing repos: %v", err)
 		}
 		g.rate = &resp.Rate
 		for _, review := range reviews {
 			r := convertGithubReview(review)
-			r.IssueIndex = pullRequestNumber
+			r.IssueIndex = context.LocalID()
 			// retrieve all review comments
 			opt2 := &github.ListOptions{
 				PerPage: g.maxPerPage,
 			}
 			for {
 				g.sleep()
-				reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), review.GetID(), opt2)
+				reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2)
 				if err != nil {
 					return nil, fmt.Errorf("error while listing repos: %v", err)
 				}

modules/migrations/github_test.go

@@ -216,7 +216,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
 
 	// downloader.GetComments()
 	comments, _, err := downloader.GetComments(base.GetCommentOptions{
-		IssueNumber: 2,
+		Context: base.BasicIssueContext(2),
 	})
 	assert.NoError(t, err)
 	assertCommentsEqual(t, []*base.Comment{
@@ -286,6 +286,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
 			Merged:         true,
 			MergedTime:     timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
 			MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
+			Context:        base.BasicIssueContext(3),
 		},
 		{
 			Number:     4,
@@ -332,10 +333,11 @@ func TestGitHubDownloadRepo(t *testing.T) {
 					Content:  "+1",
 				},
 			},
+			Context: base.BasicIssueContext(4),
 		},
 	}, prs)
 
-	reviews, err := downloader.GetReviews(3)
+	reviews, err := downloader.GetReviews(base.BasicIssueContext(3))
 	assert.NoError(t, err)
 	assertReviewsEqual(t, []*base.Review{
 		{
@@ -367,7 +369,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
 		},
 	}, reviews)
 
-	reviews, err = downloader.GetReviews(4)
+	reviews, err = downloader.GetReviews(base.BasicIssueContext(4))
 	assert.NoError(t, err)
 	assertReviewsEqual(t, []*base.Review{
 		{