From 6f513103c9e63b8a09ad00e1599206581eb08a24 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 10 Jul 2022 17:03:49 -0400 Subject: [PATCH 01/94] Define interfaces syncing new issues/PRs --- modules/migration/downloader.go | 5 +++++ modules/migration/null_downloader.go | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go index 7759c96056a92..02f01987cea6d 100644 --- a/modules/migration/downloader.go +++ b/modules/migration/downloader.go @@ -7,6 +7,7 @@ package migration import ( "context" + "time" "code.gitea.io/gitea/modules/structs" ) @@ -26,6 +27,10 @@ type Downloader interface { GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) GetReviews(reviewable Reviewable) ([]*Review, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) + + // For syncing issues and pull requests + GetNewIssues(page, perPage int, updatedAfter time.Time) ([]*Issue, bool, error) + GetNewPullRequests(page, perPage int, updatedAfter time.Time) ([]*PullRequest, bool, error) } // DownloaderFactory defines an interface to match a downloader implementation and create a downloader diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go index ad925c32ce3cf..be2d28923d024 100644 --- a/modules/migration/null_downloader.go +++ b/modules/migration/null_downloader.go @@ -7,6 +7,7 @@ package migration import ( "context" "net/url" + "time" ) // NullDownloader implements a blank downloader @@ -87,3 +88,13 @@ func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) ( func (n NullDownloader) SupportGetRepoComments() bool { return false } + +// GetNewIssues returns new issues updated after the given time according start and limit +func (n NullDownloader) GetNewIssues(page, perPage int, updatedAfter time.Time) ([]*Issue, bool, error) { + return nil, false, ErrNotSupported{Entity: "Issues"} +} + +// GetNewPullRequests returns pull requests after the given time according page and perPage +func (n NullDownloader) GetNewPullRequests(page, perPage int, updatedAfter time.Time) ([]*PullRequest, bool, error) { + return nil, false, ErrNotSupported{Entity: "PullRequests"} +} From 2bcb4c90fda5f0b3b95788cc0a5f2b92a10e0b64 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 10 Jul 2022 17:48:40 -0400 Subject: [PATCH 02/94] Support getting new issues from github --- services/migrations/github.go | 177 ++++++++++++++++++---------------- 1 file changed, 94 insertions(+), 83 deletions(-) diff --git a/services/migrations/github.go b/services/migrations/github.go index 5f5b430fa9e07..97671e36d2605 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -383,89 +383,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { // GetIssues returns issues according start and limit func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { - if perPage > g.maxPerPage { - perPage = g.maxPerPage - } - opt := &github.IssueListByRepoOptions{ - Sort: "created", - Direction: "asc", - State: "all", - ListOptions: github.ListOptions{ - PerPage: perPage, - Page: page, - }, - } - - allIssues := make([]*base.Issue, 0, perPage) - g.waitAndPickClient() - issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) - if err != nil { - return nil, false, fmt.Errorf("error while listing repos: %v", err) - } - log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues)) - g.setRate(&resp.Rate) - for _, issue := range issues { - if issue.IsPullRequest() { - continue - } - - labels := make([]*base.Label, 0, len(issue.Labels)) - for _, l := range issue.Labels { - labels = append(labels, convertGithubLabel(l)) - } - - // get reactions - var reactions []*base.Reaction - if !g.SkipReactions { - for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{ - Page: i, - PerPage: perPage, - }) - if err != nil { - return nil, false, err - } - g.setRate(&resp.Rate) - if len(res) == 0 { - break - } - for _, reaction := range res { - reactions = append(reactions, &base.Reaction{ - UserID: reaction.User.GetID(), - UserName: reaction.User.GetLogin(), - Content: reaction.GetContent(), - }) - } - } - } - - var assignees []string - for i := range issue.Assignees { - assignees = append(assignees, issue.Assignees[i].GetLogin()) - } - - allIssues = append(allIssues, &base.Issue{ - Title: *issue.Title, - Number: int64(*issue.Number), - PosterID: issue.GetUser().GetID(), - PosterName: issue.GetUser().GetLogin(), - PosterEmail: issue.GetUser().GetEmail(), - Content: issue.GetBody(), - Milestone: issue.GetMilestone().GetTitle(), - State: issue.GetState(), - Created: issue.GetCreatedAt(), - Updated: issue.GetUpdatedAt(), - Labels: labels, - Reactions: reactions, - Closed: issue.ClosedAt, - IsLocked: issue.GetLocked(), - Assignees: assignees, - ForeignIndex: int64(*issue.Number), - }) - } - - return allIssues, len(issues) < perPage, nil + return g.getIssuesSince(page, perPage, time.Time{}) // set since to empty to get all issues } // SupportGetRepoComments return true if it supports get repo comments @@ -843,3 +761,96 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev } return allReviews, nil } + +// GetNewIssues returns new issues updated after the given time according start and limit +func (g *GithubDownloaderV3) GetNewIssues(page, perPage int, updatedAfter time.Time) ([]*base.Issue, bool, error) { + return g.getIssuesSince(page, perPage, updatedAfter) +} + +// getIssuesSince returns issues given page, perPage and since +func (g *GithubDownloaderV3) getIssuesSince(page, perPage int, since time.Time) ([]*base.Issue, bool, error) { + if perPage > g.maxPerPage { + perPage = g.maxPerPage + } + opt := &github.IssueListByRepoOptions{ + Sort: "created", + Direction: "asc", + State: "all", + Since: since, + ListOptions: github.ListOptions{ + PerPage: perPage, + Page: page, + }, + } + + allIssues := make([]*base.Issue, 0, perPage) + g.waitAndPickClient() + issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) + if err != nil { + return nil, false, fmt.Errorf("error while listing repos: %v", err) + } + log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues)) + g.setRate(&resp.Rate) + for _, issue := range issues { + if issue.IsPullRequest() { + continue + } + + labels := make([]*base.Label, 0, len(issue.Labels)) + for _, l := range issue.Labels { + labels = append(labels, convertGithubLabel(l)) + } + + // get reactions + var reactions []*base.Reaction + if !g.SkipReactions { + for i := 1; ; i++ { + g.waitAndPickClient() + res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{ + Page: i, + PerPage: perPage, + }) + if err != nil { + return nil, false, err + } + g.setRate(&resp.Rate) + if len(res) == 0 { + break + } + for _, reaction := range res { + reactions = append(reactions, &base.Reaction{ + UserID: reaction.User.GetID(), + UserName: reaction.User.GetLogin(), + Content: reaction.GetContent(), + }) + } + } + } + + var assignees []string + for i := range issue.Assignees { + assignees = append(assignees, issue.Assignees[i].GetLogin()) + } + + allIssues = append(allIssues, &base.Issue{ + Title: *issue.Title, + Number: int64(*issue.Number), + PosterID: issue.GetUser().GetID(), + PosterName: issue.GetUser().GetLogin(), + PosterEmail: issue.GetUser().GetEmail(), + Content: issue.GetBody(), + Milestone: issue.GetMilestone().GetTitle(), + State: issue.GetState(), + Created: issue.GetCreatedAt(), + Updated: issue.GetUpdatedAt(), + Labels: labels, + Reactions: reactions, + Closed: issue.ClosedAt, + IsLocked: issue.GetLocked(), + Assignees: assignees, + ForeignIndex: int64(*issue.Number), + }) + } + + return allIssues, len(issues) < perPage, nil +} From c54aa4e00e64d231ec2952c3ee7c46aad622b2bd Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 10 Jul 2022 18:13:43 -0400 Subject: [PATCH 03/94] Put issue reactions code in a seperate function --- services/migrations/github.go | 83 ++++++++++++++++------------------- 1 file changed, 37 insertions(+), 46 deletions(-) diff --git a/services/migrations/github.go b/services/migrations/github.go index 97671e36d2605..7dfd9308dd8c8 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -565,29 +565,9 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq } // get reactions - var reactions []*base.Reaction - if !g.SkipReactions { - for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{ - Page: i, - PerPage: perPage, - }) - if err != nil { - return nil, false, err - } - g.setRate(&resp.Rate) - if len(res) == 0 { - break - } - for _, reaction := range res { - reactions = append(reactions, &base.Reaction{ - UserID: reaction.User.GetID(), - UserName: reaction.User.GetLogin(), - Content: reaction.GetContent(), - }) - } - } + reactions, err := g.getReactions(pr.GetNumber(), perPage) + if err != nil { + return nil, false, err } // download patch and saved as tmp file @@ -802,29 +782,9 @@ func (g *GithubDownloaderV3) getIssuesSince(page, perPage int, since time.Time) } // get reactions - var reactions []*base.Reaction - if !g.SkipReactions { - for i := 1; ; i++ { - g.waitAndPickClient() - res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{ - Page: i, - PerPage: perPage, - }) - if err != nil { - return nil, false, err - } - g.setRate(&resp.Rate) - if len(res) == 0 { - break - } - for _, reaction := range res { - reactions = append(reactions, &base.Reaction{ - UserID: reaction.User.GetID(), - UserName: reaction.User.GetLogin(), - Content: reaction.GetContent(), - }) - } - } + reactions, err := g.getReactions(issue.GetNumber(), perPage) + if err != nil { + return nil, false, err } var assignees []string @@ -854,3 +814,34 @@ func (g *GithubDownloaderV3) getIssuesSince(page, perPage int, since time.Time) return allIssues, len(issues) < perPage, nil } + +func (g *GithubDownloaderV3) getReactions(number, perPage int) ([]*base.Reaction, error) { + if g.SkipReactions { + return nil, nil + } + + var reactions []*base.Reaction + + for i := 1; ; i++ { + g.waitAndPickClient() + res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, number, &github.ListOptions{ + Page: i, + PerPage: perPage, + }) + if err != nil { + return nil, err + } + g.setRate(&resp.Rate) + if len(res) == 0 { + break + } + for _, reaction := range res { + reactions = append(reactions, &base.Reaction{ + UserID: reaction.User.GetID(), + UserName: reaction.User.GetLogin(), + Content: reaction.GetContent(), + }) + } + } + return reactions, nil +} From 8532fa38bff68ef3a173b596ec4f814d36b74a29 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 10 Jul 2022 18:16:58 -0400 Subject: [PATCH 04/94] Rename function to getIssueReactions --- services/migrations/github.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/migrations/github.go b/services/migrations/github.go index 7dfd9308dd8c8..277cb607ad759 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -565,7 +565,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq } // get reactions - reactions, err := g.getReactions(pr.GetNumber(), perPage) + reactions, err := g.getIssueReactions(pr.GetNumber(), perPage) if err != nil { return nil, false, err } @@ -782,7 +782,7 @@ func (g *GithubDownloaderV3) getIssuesSince(page, perPage int, since time.Time) } // get reactions - reactions, err := g.getReactions(issue.GetNumber(), perPage) + reactions, err := g.getIssueReactions(issue.GetNumber(), perPage) if err != nil { return nil, false, err } @@ -815,7 +815,7 @@ func (g *GithubDownloaderV3) getIssuesSince(page, perPage int, since time.Time) return allIssues, len(issues) < perPage, nil } -func (g *GithubDownloaderV3) getReactions(number, perPage int) ([]*base.Reaction, error) { +func (g *GithubDownloaderV3) getIssueReactions(number, perPage int) ([]*base.Reaction, error) { if g.SkipReactions { return nil, nil } From ad8eb2b3d0bd54fc220cdbaaccf566eb49ae26a9 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 10 Jul 2022 18:35:17 -0400 Subject: [PATCH 05/94] Support getting new PRs from github --- services/migrations/github.go | 144 +++++++++++++++++++++++----------- 1 file changed, 99 insertions(+), 45 deletions(-) diff --git a/services/migrations/github.go b/services/migrations/github.go index 277cb607ad759..4abe01ba15dc0 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -559,54 +559,11 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs)) g.setRate(&resp.Rate) for _, pr := range prs { - labels := make([]*base.Label, 0, len(pr.Labels)) - for _, l := range pr.Labels { - labels = append(labels, convertGithubLabel(l)) - } - - // get reactions - reactions, err := g.getIssueReactions(pr.GetNumber(), perPage) + basePR, err := g.convertGithubPullRequest(pr, perPage) if err != nil { return nil, false, err } - - // download patch and saved as tmp file - g.waitAndPickClient() - - allPRs = append(allPRs, &base.PullRequest{ - Title: pr.GetTitle(), - Number: int64(pr.GetNumber()), - PosterID: pr.GetUser().GetID(), - PosterName: pr.GetUser().GetLogin(), - PosterEmail: pr.GetUser().GetEmail(), - Content: pr.GetBody(), - Milestone: pr.GetMilestone().GetTitle(), - State: pr.GetState(), - Created: pr.GetCreatedAt(), - Updated: pr.GetUpdatedAt(), - Closed: pr.ClosedAt, - Labels: labels, - Merged: pr.MergedAt != nil, - MergeCommitSHA: pr.GetMergeCommitSHA(), - MergedTime: pr.MergedAt, - IsLocked: pr.ActiveLockReason != nil, - Head: base.PullRequestBranch{ - Ref: pr.GetHead().GetRef(), - SHA: pr.GetHead().GetSHA(), - OwnerName: pr.GetHead().GetUser().GetLogin(), - RepoName: pr.GetHead().GetRepo().GetName(), - CloneURL: pr.GetHead().GetRepo().GetCloneURL(), - }, - Base: base.PullRequestBranch{ - Ref: pr.GetBase().GetRef(), - SHA: pr.GetBase().GetSHA(), - RepoName: pr.GetBase().GetRepo().GetName(), - OwnerName: pr.GetBase().GetUser().GetLogin(), - }, - PatchURL: pr.GetPatchURL(), - Reactions: reactions, - ForeignIndex: int64(*pr.Number), - }) + allPRs = append(allPRs, basePR) } return allPRs, len(prs) < perPage, nil @@ -815,6 +772,103 @@ func (g *GithubDownloaderV3) getIssuesSince(page, perPage int, since time.Time) return allIssues, len(issues) < perPage, nil } +// GetNewPullRequests returns pull requests after the given time according page and perPage +func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter time.Time) ([]*base.PullRequest, bool, error) { + // Every pull request is an issue, and only Issues API provides parameter `since`, + // So we should get issues IDs first and then get pull requests + if perPage > g.maxPerPage { + perPage = g.maxPerPage + } + opt := &github.IssueListByRepoOptions{ + Sort: "created", + Direction: "asc", + State: "all", + Since: updatedAfter, + ListOptions: github.ListOptions{ + PerPage: perPage, + Page: page, + }, + } + + allPRs := make([]*base.PullRequest, 0, perPage) + g.waitAndPickClient() + issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) + if err != nil { + return nil, false, fmt.Errorf("error while listing repos: %v", err) + } + log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues)) + g.setRate(&resp.Rate) + for _, issue := range issues { + if !issue.IsPullRequest() { + continue + } + + pr, resp, err := g.getClient().PullRequests.Get(g.ctx, g.repoOwner, g.repoName, issue.GetNumber()) + if err != nil { + return nil, false, fmt.Errorf("error while getting repo pull request: %v", err) + } + g.setRate(&resp.Rate) + basePR, err := g.convertGithubPullRequest(pr, perPage) + if err != nil { + return nil, false, err + } + allPRs = append(allPRs, basePR) + } + + return nil, false, nil +} + +func (g *GithubDownloaderV3) convertGithubPullRequest(pr *github.PullRequest, perPage int) (*base.PullRequest, error) { + labels := make([]*base.Label, 0, len(pr.Labels)) + for _, l := range pr.Labels { + labels = append(labels, convertGithubLabel(l)) + } + + // get reactions + reactions, err := g.getIssueReactions(pr.GetNumber(), perPage) + if err != nil { + return nil, err + } + + // download patch and saved as tmp file + g.waitAndPickClient() + + return &base.PullRequest{ + Title: pr.GetTitle(), + Number: int64(pr.GetNumber()), + PosterID: pr.GetUser().GetID(), + PosterName: pr.GetUser().GetLogin(), + PosterEmail: pr.GetUser().GetEmail(), + Content: pr.GetBody(), + Milestone: pr.GetMilestone().GetTitle(), + State: pr.GetState(), + Created: pr.GetCreatedAt(), + Updated: pr.GetUpdatedAt(), + Closed: pr.ClosedAt, + Labels: labels, + Merged: pr.MergedAt != nil, + MergeCommitSHA: pr.GetMergeCommitSHA(), + MergedTime: pr.MergedAt, + IsLocked: pr.ActiveLockReason != nil, + Head: base.PullRequestBranch{ + Ref: pr.GetHead().GetRef(), + SHA: pr.GetHead().GetSHA(), + OwnerName: pr.GetHead().GetUser().GetLogin(), + RepoName: pr.GetHead().GetRepo().GetName(), + CloneURL: pr.GetHead().GetRepo().GetCloneURL(), + }, + Base: base.PullRequestBranch{ + Ref: pr.GetBase().GetRef(), + SHA: pr.GetBase().GetSHA(), + RepoName: pr.GetBase().GetRepo().GetName(), + OwnerName: pr.GetBase().GetUser().GetLogin(), + }, + PatchURL: pr.GetPatchURL(), + Reactions: reactions, + ForeignIndex: int64(*pr.Number), + }, nil +} + func (g *GithubDownloaderV3) getIssueReactions(number, perPage int) ([]*base.Reaction, error) { if g.SkipReactions { return nil, nil From 40c3b32ea5ae56901e1ad50991c93d6f86890d10 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 10 Jul 2022 18:40:29 -0400 Subject: [PATCH 06/94] Fix issue that new PRs not returned --- services/migrations/github.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/migrations/github.go b/services/migrations/github.go index 4abe01ba15dc0..cb218ed33e3b7 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -815,7 +815,7 @@ func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter allPRs = append(allPRs, basePR) } - return nil, false, nil + return allPRs, len(issues) < perPage, nil } func (g *GithubDownloaderV3) convertGithubPullRequest(pr *github.PullRequest, perPage int) (*base.PullRequest, error) { From 039e505ec9fc6283626a65b0dc4ee088debc0701 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 10 Jul 2022 22:59:03 -0400 Subject: [PATCH 07/94] Allow migration items with mirroring for GitHub --- routers/api/v1/repo/migrate.go | 8 -------- web_src/js/features/repo-migration.js | 3 ++- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index f868c53951a16..0d0e3ab7aefa4 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -161,14 +161,6 @@ func Migrate(ctx *context.APIContext) { GitServiceType: gitServiceType, MirrorInterval: form.MirrorInterval, } - if opts.Mirror { - opts.Issues = false - opts.Milestones = false - opts.Labels = false - opts.Comments = false - opts.PullRequests = false - opts.Releases = false - } repo, err := repo_module.CreateRepository(ctx.Doer, repoOwner, models.CreateRepoOptions{ Name: opts.RepoName, diff --git a/web_src/js/features/repo-migration.js b/web_src/js/features/repo-migration.js index ece01e53bd72f..05295eaedf554 100644 --- a/web_src/js/features/repo-migration.js +++ b/web_src/js/features/repo-migration.js @@ -44,7 +44,8 @@ function checkItems(tokenAuth) { enableItems = $user.val() !== '' || $pass.val() !== ''; } if (enableItems && $service.val() > 1) { - if ($mirror.is(':checked')) { + const allowedServices = [2]; // services that supports migration items with mirroring + if ($mirror.is(':checked') && !allowedServices.includes(Number($service.val()))) { $items.not('[name="wiki"]').attr('disabled', true); $items.filter('[name="wiki"]').attr('disabled', false); return; From a3ca5b0896eb9dc69ec9fd2ecd659747b6e7c866 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 10 Jul 2022 23:00:06 -0400 Subject: [PATCH 08/94] Fix a comment --- modules/notification/notification.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/notification/notification.go b/modules/notification/notification.go index bdfed90b7864e..37064ecc4a765 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -227,7 +227,7 @@ func NotifyCreateRepository(doer, u *user_model.User, repo *repo_model.Repositor } } -// NotifyMigrateRepository notifies create repository to notifiers +// NotifyMigrateRepository notifies migrate repository to notifiers func NotifyMigrateRepository(doer, u *user_model.User, repo *repo_model.Repository) { for _, notifier := range notifiers { notifier.NotifyMigrateRepository(doer, u, repo) From 176009ed60ec08eb0fec010d962a73d11e301605 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Mon, 11 Jul 2022 01:54:09 -0400 Subject: [PATCH 09/94] Create relevant functions for uploader --- modules/migration/null_uploader.go | 96 +++++++++++++++++++++++++++ modules/migration/uploader.go | 8 +++ services/migrations/dump.go | 1 + services/migrations/gitea_uploader.go | 34 ++++++++++ 4 files changed, 139 insertions(+) create mode 100644 modules/migration/null_uploader.go diff --git a/modules/migration/null_uploader.go b/modules/migration/null_uploader.go new file mode 100644 index 0000000000000..1a4e6920ebd5f --- /dev/null +++ b/modules/migration/null_uploader.go @@ -0,0 +1,96 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migration + +// NullUploader implements a blank uploader +type NullUploader struct{} + +var _ Downloader = &NullDownloader{} + +func (g *NullUploader) MaxBatchInsertSize(tp string) int { + return 0 +} + +func (g *NullUploader) CreateRepo(repo *Repository, opts MigrateOptions) error { + return nil +} + +func (g *NullUploader) CreateTopics(topic ...string) error { + return nil +} + +func (g *NullUploader) CreateMilestones(milestones ...*Milestone) error { + return nil +} + +func (g *NullUploader) CreateReleases(releases ...*Release) error { + return nil +} + +func (g *NullUploader) SyncTags() error { + return nil +} + +func (g *NullUploader) CreateLabels(labels ...*Label) error { + return nil +} + +func (g *NullUploader) CreateIssues(issues ...*Issue) error { + return nil +} + +func (g *NullUploader) CreateComments(comments ...*Comment) error { + return nil +} + +func (g *NullUploader) CreatePullRequests(prs ...*PullRequest) error { + return nil +} + +func (g *NullUploader) CreateReviews(reviews ...*Review) error { + return nil +} + +func (g *NullUploader) UpdateTopics(topic ...string) error { + return nil +} + +func (g *NullUploader) UpdateMilestones(milestones ...*Milestone) error { + return nil +} + +func (g *NullUploader) UpdateReleases(releases ...*Release) error { + return nil +} + +func (g *NullUploader) UpdateLabels(labels ...*Label) error { + return nil +} + +func (g *NullUploader) UpdateIssues(issues ...*Issue) error { + return nil +} + +func (g *NullUploader) UpdateComments(comments ...*Comment) error { + return nil +} + +func (g *NullUploader) UpdatePullRequests(prs ...*PullRequest) error { + return nil +} + +func (g *NullUploader) UpdateReviews(reviews ...*Review) error { + return nil +} + +func (g *NullUploader) Rollback() error { + return nil +} + +func (g *NullUploader) Finish() error { + return nil +} + +func (g *NullUploader) Close() {} diff --git a/modules/migration/uploader.go b/modules/migration/uploader.go index 57571861aaad8..a0704a0df5e0e 100644 --- a/modules/migration/uploader.go +++ b/modules/migration/uploader.go @@ -18,6 +18,14 @@ type Uploader interface { CreateComments(comments ...*Comment) error CreatePullRequests(prs ...*PullRequest) error CreateReviews(reviews ...*Review) error + UpdateTopics(topic ...string) error + UpdateMilestones(milestones ...*Milestone) error + UpdateReleases(releases ...*Release) error + UpdateLabels(labels ...*Label) error + UpdateIssues(issues ...*Issue) error + UpdateComments(comments ...*Comment) error + UpdatePullRequests(prs ...*PullRequest) error + UpdateReviews(reviews ...*Review) error Rollback() error Finish() error Close() diff --git a/services/migrations/dump.go b/services/migrations/dump.go index a9ec459519e56..bd19302b96aa4 100644 --- a/services/migrations/dump.go +++ b/services/migrations/dump.go @@ -33,6 +33,7 @@ var _ base.Uploader = &RepositoryDumper{} // RepositoryDumper implements an Uploader to the local directory type RepositoryDumper struct { + base.NullUploader ctx context.Context baseDir string repoOwner string diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index e71b2ca17af35..fb302b4f83f4f 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -40,6 +40,7 @@ var _ base.Uploader = &GiteaLocalUploader{} // GiteaLocalUploader implements an Uploader to gitea sites type GiteaLocalUploader struct { + base.NullUploader ctx context.Context doer *user_model.User repoOwner string @@ -801,6 +802,39 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { return issues_model.InsertReviews(cms) } +// UpdateTopics updates topics +func (g *GiteaLocalUploader) UpdateTopics(topics ...string) error { + return g.CreateTopics(topics...) +} + +func (g *GiteaLocalUploader) UpdateMilestones(milestones ...*base.Milestone) error { + return nil +} + +func (g *GiteaLocalUploader) UpdateReleases(releases ...*base.Release) error { + return nil +} + +func (g *GiteaLocalUploader) UpdateLabels(labels ...*base.Label) error { + return nil +} + +func (g *GiteaLocalUploader) UpdateIssues(issues ...*base.Issue) error { + return nil +} + +func (g *GiteaLocalUploader) UpdateComments(comments ...*base.Comment) error { + return nil +} + +func (g *GiteaLocalUploader) UpdatePullRequests(prs ...*base.PullRequest) error { + return nil +} + +func (g *GiteaLocalUploader) UpdateReviews(reviews ...*base.Review) error { + return nil +} + // Rollback when migrating failed, this will rollback all the changes. func (g *GiteaLocalUploader) Rollback() error { if g.repo != nil && g.repo.ID > 0 { From 698d6be490f97d02b3e0985c3202c9e5580d7171 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Mon, 11 Jul 2022 04:17:45 -0400 Subject: [PATCH 10/94] Optimize topics creation and updates --- models/repo/topic.go | 30 ++++++++++++++++++++++++++- services/migrations/gitea_uploader.go | 16 +++++++++----- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/models/repo/topic.go b/models/repo/topic.go index 2a16467215d3a..24ce33dcd0f08 100644 --- a/models/repo/topic.go +++ b/models/repo/topic.go @@ -262,6 +262,34 @@ func AddTopic(repoID int64, topicName string) (*Topic, error) { return topic, committer.Commit() } +func AddTopics(repoID int64, topicNames ...string) error { + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + sess := db.GetEngine(ctx) + + for _, topicName := range topicNames { + if strings.TrimSpace(topicName) == "" { + continue + } + + _, err := addTopicByNameToRepo(ctx, repoID, topicName) + if err != nil { + return err + } + } + + if _, err := sess.ID(repoID).Cols("topics").Update(&Repository{ + Topics: topicNames, + }); err != nil { + return err + } + + return committer.Commit() +} + // DeleteTopic removes a topic name from a repository (if it has it) func DeleteTopic(repoID int64, topicName string) (*Topic, error) { topic, err := GetRepoTopicByName(db.DefaultContext, repoID, topicName) @@ -278,7 +306,7 @@ func DeleteTopic(repoID int64, topicName string) (*Topic, error) { return topic, err } -// SaveTopics save topics to a repository +// SaveTopics save topics to a repository (add and delete respective topics) func SaveTopics(repoID int64, topicNames ...string) error { topics, _, err := FindTopics(&FindTopicOptions{ RepoID: repoID, diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index fb302b4f83f4f..e21d23126e00e 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -150,9 +150,8 @@ func (g *GiteaLocalUploader) Close() { } } -// CreateTopics creates topics -func (g *GiteaLocalUploader) CreateTopics(topics ...string) error { - // ignore topics to long for the db +func filterTopicsForDB(topics []string) []string { + // filter out topics to long for the db c := 0 for i := range topics { if len(topics[i]) <= 50 { @@ -161,7 +160,13 @@ func (g *GiteaLocalUploader) CreateTopics(topics ...string) error { } } topics = topics[:c] - return repo_model.SaveTopics(g.repo.ID, topics...) + return topics +} + +// CreateTopics creates topics +func (g *GiteaLocalUploader) CreateTopics(topics ...string) error { + topics = filterTopicsForDB(topics) + return repo_model.AddTopics(g.repo.ID, topics...) } // CreateMilestones creates milestones @@ -804,7 +809,8 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { // UpdateTopics updates topics func (g *GiteaLocalUploader) UpdateTopics(topics ...string) error { - return g.CreateTopics(topics...) + topics = filterTopicsForDB(topics) + return repo_model.SaveTopics(g.repo.ID, topics...) } func (g *GiteaLocalUploader) UpdateMilestones(milestones ...*base.Milestone) error { From fb82294fc60624374106e31d1240775695fb2e6f Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Mon, 11 Jul 2022 23:20:40 -0400 Subject: [PATCH 11/94] Support updating labels --- models/issues/label.go | 96 ++++++++++++++++++++++++--- services/migrations/gitea_uploader.go | 30 ++++++--- 2 files changed, 107 insertions(+), 19 deletions(-) diff --git a/models/issues/label.go b/models/issues/label.go index 9ad488252ab7f..82222ef865e89 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -237,27 +237,39 @@ func NewLabels(labels ...*Label) error { // UpdateLabel updates label information. func UpdateLabel(l *Label) error { + return updateLabel(db.DefaultContext, l) +} + +func updateLabel(ctx context.Context, l *Label) error { if !LabelColorPattern.MatchString(l.Color) { return fmt.Errorf("bad color code: %s", l.Color) } - return updateLabelCols(db.DefaultContext, l, "name", "description", "color") + return updateLabelCols(ctx, l, "name", "description", "color") } // DeleteLabel delete a label func DeleteLabel(id, labelID int64) error { - label, err := GetLabelByID(db.DefaultContext, labelID) + ctx, committer, err := db.TxContext() if err != nil { - if IsErrLabelNotExist(err) { - return nil - } return err } + defer committer.Close() - ctx, committer, err := db.TxContext() + if err = deleteLabel(ctx, id, labelID); err != nil { + return err + } + + return committer.Commit() +} + +func deleteLabel(ctx context.Context, id, labelID int64) error { + label, err := GetLabelByID(ctx, labelID) if err != nil { + if IsErrLabelNotExist(err) { + return nil + } return err } - defer committer.Close() sess := db.GetEngine(ctx) @@ -281,7 +293,7 @@ func DeleteLabel(id, labelID int64) error { return err } - return committer.Commit() + return nil } // GetLabelByID returns a label by given ID. @@ -416,6 +428,72 @@ func GetLabelsByRepoID(ctx context.Context, repoID int64, sortType string, listO return labels, sess.Find(&labels) } +// UpdateLabels adds, updates, and deletes relevant labels for the given repository. +func UpdateLabelsByRepoID(repoID int64, labels ...*Label) error { + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + + existingLabels, err := GetLabelsByRepoID(ctx, repoID, "", db.ListOptions{}) + labelsToAdd := make([]*Label, 0) + labelsToUpdate := make([]*Label, 0) + labelsToDelete := make([]*Label, 0) + + for _, label := range labels { + if !LabelColorPattern.MatchString(label.Color) { + return fmt.Errorf("bad color code: %s", label.Color) + } + + found := false + for _, existingLabel := range existingLabels { + if existingLabel.ID == label.ID { + found = true + if existingLabel.Name != label.Name || existingLabel.Description != label.Description || + existingLabel.Color != label.Color { + labelsToUpdate = append(labelsToUpdate, label) + } + } + } + if !found { + labelsToAdd = append(labelsToAdd, label) + } + } + + for _, existingLabel := range existingLabels { + found := false + for _, label := range labels { + if label.ID == existingLabel.ID { + found = true + } + } + if !found { + labelsToDelete = append(labelsToDelete, existingLabel) + } + } + + for _, label := range labelsToAdd { + if err = NewLabel(ctx, label); err != nil { + return err + } + } + + for _, label := range labelsToUpdate { + if err = updateLabel(ctx, label); err != nil { + return err + } + } + + for _, label := range labelsToDelete { + if err = deleteLabel(ctx, repoID, label.ID); err != nil { + return err + } + } + + return nil +} + // CountLabelsByRepoID count number of all labels that belong to given repository by ID. func CountLabelsByRepoID(repoID int64) (int64, error) { return db.GetEngine(db.DefaultContext).Where("repo_id = ?", repoID).Count(&Label{}) @@ -718,7 +796,7 @@ func DeleteIssueLabel(ctx context.Context, issue *Issue, label *Label, doer *use return issue.LoadLabels(ctx) } -// DeleteLabelsByRepoID deletes labels of some repository +// DeleteLabelsByRepoID deletes labels of some repository func DeleteLabelsByRepoID(ctx context.Context, repoID int64) error { deleteCond := builder.Select("id").From("label").Where(builder.Eq{"label.repo_id": repoID}) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index e21d23126e00e..2f3d6fe1ca3c7 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -222,24 +222,27 @@ func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) err // CreateLabels creates labels func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error { + lbs := convertLabels(g.repo.ID, labels...) + if err := issues_model.NewLabels(lbs...); err != nil { + return err + } + for _, lb := range lbs { + g.labels[lb.Name] = lb + } + return nil +} + +func convertLabels(repoID int64, labels ...*base.Label) []*issues_model.Label { lbs := make([]*issues_model.Label, 0, len(labels)) for _, label := range labels { lbs = append(lbs, &issues_model.Label{ - RepoID: g.repo.ID, + RepoID: repoID, Name: label.Name, Description: label.Description, Color: fmt.Sprintf("#%s", label.Color), }) } - - err := issues_model.NewLabels(lbs...) - if err != nil { - return err - } - for _, lb := range lbs { - g.labels[lb.Name] = lb - } - return nil + return lbs } // CreateReleases creates releases @@ -822,6 +825,13 @@ func (g *GiteaLocalUploader) UpdateReleases(releases ...*base.Release) error { } func (g *GiteaLocalUploader) UpdateLabels(labels ...*base.Label) error { + lbs := convertLabels(g.repo.ID, labels...) + if err := issues_model.UpdateLabelsByRepoID(g.repo.ID, lbs...); err != nil { + return err + } + for _, lb := range lbs { + g.labels[lb.Name] = lb + } return nil } From 22835e43754264408d6d550f50820aa161080a27 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Mon, 11 Jul 2022 23:25:43 -0400 Subject: [PATCH 12/94] Fix error not returned error --- models/issues/label.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/issues/label.go b/models/issues/label.go index 82222ef865e89..522b237bbb9c6 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -437,6 +437,9 @@ func UpdateLabelsByRepoID(repoID int64, labels ...*Label) error { defer committer.Close() existingLabels, err := GetLabelsByRepoID(ctx, repoID, "", db.ListOptions{}) + if err != nil { + return err + } labelsToAdd := make([]*Label, 0) labelsToUpdate := make([]*Label, 0) labelsToDelete := make([]*Label, 0) From 98a8e699bd7256e43a5b4237669f1263b8272a3e Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Mon, 11 Jul 2022 23:39:32 -0400 Subject: [PATCH 13/94] Update Uploader interface --- modules/migration/null_uploader.go | 14 +++++++------- modules/migration/uploader.go | 16 ++++++++-------- services/migrations/gitea_uploader.go | 16 ++++++++-------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/modules/migration/null_uploader.go b/modules/migration/null_uploader.go index 1a4e6920ebd5f..97ceceba9fd30 100644 --- a/modules/migration/null_uploader.go +++ b/modules/migration/null_uploader.go @@ -7,7 +7,7 @@ package migration // NullUploader implements a blank uploader type NullUploader struct{} -var _ Downloader = &NullDownloader{} +var _ Uploader = &NullUploader{} func (g *NullUploader) MaxBatchInsertSize(tp string) int { return 0 @@ -61,27 +61,27 @@ func (g *NullUploader) UpdateMilestones(milestones ...*Milestone) error { return nil } -func (g *NullUploader) UpdateReleases(releases ...*Release) error { +func (g *NullUploader) UpdateLabels(labels ...*Label) error { return nil } -func (g *NullUploader) UpdateLabels(labels ...*Label) error { +func (g *NullUploader) PatchReleases(releases ...*Release) error { return nil } -func (g *NullUploader) UpdateIssues(issues ...*Issue) error { +func (g *NullUploader) PatchIssues(issues ...*Issue) error { return nil } -func (g *NullUploader) UpdateComments(comments ...*Comment) error { +func (g *NullUploader) PatchComments(comments ...*Comment) error { return nil } -func (g *NullUploader) UpdatePullRequests(prs ...*PullRequest) error { +func (g *NullUploader) PatchPullRequests(prs ...*PullRequest) error { return nil } -func (g *NullUploader) UpdateReviews(reviews ...*Review) error { +func (g *NullUploader) PatchReviews(reviews ...*Review) error { return nil } diff --git a/modules/migration/uploader.go b/modules/migration/uploader.go index a0704a0df5e0e..9a87778f4dfe8 100644 --- a/modules/migration/uploader.go +++ b/modules/migration/uploader.go @@ -18,14 +18,14 @@ type Uploader interface { CreateComments(comments ...*Comment) error CreatePullRequests(prs ...*PullRequest) error CreateReviews(reviews ...*Review) error - UpdateTopics(topic ...string) error - UpdateMilestones(milestones ...*Milestone) error - UpdateReleases(releases ...*Release) error - UpdateLabels(labels ...*Label) error - UpdateIssues(issues ...*Issue) error - UpdateComments(comments ...*Comment) error - UpdatePullRequests(prs ...*PullRequest) error - UpdateReviews(reviews ...*Review) error + UpdateTopics(topic ...string) error // update topics of a repository, and delete unused ones + UpdateMilestones(milestones ...*Milestone) error // update milestones of a repository, and delete unused ones + UpdateLabels(labels ...*Label) error // rewrite all issue labels and delete unused ones + PatchReleases(releases ...*Release) error // add or update releases (no deletes) + PatchComments(comments ...*Comment) error // add or update comments (no deletes) + PatchIssues(issues ...*Issue) error // add or update issues (no deletes) + PatchPullRequests(prs ...*PullRequest) error // add or update pull requests (no deletes) + PatchReviews(reviews ...*Review) error // add or update reviews (no deletes) Rollback() error Finish() error Close() diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 2f3d6fe1ca3c7..5179017afd446 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -820,10 +820,6 @@ func (g *GiteaLocalUploader) UpdateMilestones(milestones ...*base.Milestone) err return nil } -func (g *GiteaLocalUploader) UpdateReleases(releases ...*base.Release) error { - return nil -} - func (g *GiteaLocalUploader) UpdateLabels(labels ...*base.Label) error { lbs := convertLabels(g.repo.ID, labels...) if err := issues_model.UpdateLabelsByRepoID(g.repo.ID, lbs...); err != nil { @@ -835,19 +831,23 @@ func (g *GiteaLocalUploader) UpdateLabels(labels ...*base.Label) error { return nil } -func (g *GiteaLocalUploader) UpdateIssues(issues ...*base.Issue) error { +func (g *GiteaLocalUploader) PatchReleases(releases ...*base.Release) error { + return nil +} + +func (g *GiteaLocalUploader) PatchIssues(issues ...*base.Issue) error { return nil } -func (g *GiteaLocalUploader) UpdateComments(comments ...*base.Comment) error { +func (g *GiteaLocalUploader) PatchComments(comments ...*base.Comment) error { return nil } -func (g *GiteaLocalUploader) UpdatePullRequests(prs ...*base.PullRequest) error { +func (g *GiteaLocalUploader) PatchPullRequests(prs ...*base.PullRequest) error { return nil } -func (g *GiteaLocalUploader) UpdateReviews(reviews ...*base.Review) error { +func (g *GiteaLocalUploader) PatchReviews(reviews ...*base.Review) error { return nil } From 7b7b166f88f5a8004d0c26f15210f1b625ea38e6 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 12 Jul 2022 02:49:49 -0400 Subject: [PATCH 14/94] Support updating issues --- models/migrate.go | 115 ++++++++++++++++++++++++-- services/migrations/gitea_uploader.go | 45 +++++++--- 2 files changed, 142 insertions(+), 18 deletions(-) diff --git a/models/migrate.go b/models/migrate.go index 0af3891cb858e..578ddf8bd0476 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -8,6 +8,7 @@ import ( "context" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/foreignreference" issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/modules/structs" ) @@ -54,18 +55,23 @@ func InsertIssues(issues ...*issues_model.Issue) error { return committer.Commit() } +func resolveIssueLabels(issueID int64, labels []*issues_model.Label) []issues_model.IssueLabel { + issueLabels := make([]issues_model.IssueLabel, 0, len(labels)) + for _, label := range labels { + issueLabels = append(issueLabels, issues_model.IssueLabel{ + IssueID: issueID, + LabelID: label.ID, + }) + } + return issueLabels +} + func insertIssue(ctx context.Context, issue *issues_model.Issue) error { sess := db.GetEngine(ctx) if _, err := sess.NoAutoTime().Insert(issue); err != nil { return err } - issueLabels := make([]issues_model.IssueLabel, 0, len(issue.Labels)) - for _, label := range issue.Labels { - issueLabels = append(issueLabels, issues_model.IssueLabel{ - IssueID: issue.ID, - LabelID: label.ID, - }) - } + issueLabels := resolveIssueLabels(issue.ID, issue.Labels) if len(issueLabels) > 0 { if _, err := sess.Insert(issueLabels); err != nil { return err @@ -92,6 +98,101 @@ func insertIssue(ctx context.Context, issue *issues_model.Issue) error { return nil } +// UpsertIssues creates new issues and updates existing issues in database +func UpsertIssues(issues ...*issues_model.Issue) error { + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + + for _, issue := range issues { + if err := upsertIssue(ctx, issue); err != nil { + return err + } + } + return committer.Commit() +} + +func updateIssue(ctx context.Context, issue *issues_model.Issue) error { + sess := db.GetEngine(ctx) + if _, err := sess.NoAutoTime().ID(issue.ID).Update(issue); err != nil { + return err + } + issueLabels := resolveIssueLabels(issue.ID, issue.Labels) + if len(issueLabels) > 0 { + labelIDs := make([]int64, 0, len(issueLabels)) + for _, label := range issueLabels { + labelIDs = append(labelIDs, label.LabelID) + } + // delete old labels + if _, err := sess.Where("issue_id = ?", issue.ID).Delete(); err != nil { + return err + } + // insert new labels + if _, err := sess.Insert(issueLabels); err != nil { + return err + } + } + + for _, reaction := range issue.Reactions { + reaction.IssueID = issue.ID + } + + if len(issue.Reactions) > 0 { + // update existing reactions and insert new ones + for _, reaction := range issue.Reactions { + exists, err := sess.Exist(&issues_model.Reaction{ID: reaction.ID}) + if err != nil { + return err + } + if exists { + if _, err := sess.ID(reaction.ID).Update(&reaction); err != nil { + return err + } + } else { + if _, err := sess.Insert(&reaction); err != nil { + return err + } + } + } + } + + if issue.ForeignReference != nil { + issue.ForeignReference.LocalIndex = issue.Index + + exists, err := sess.Exist(&foreignreference.ForeignReference{ + RepoID: issue.ForeignReference.RepoID, + LocalIndex: issue.ForeignReference.LocalIndex, + }) + if err != nil { + return err + } + + if !exists { + if _, err := sess.Insert(issue.ForeignReference); err != nil { + return err + } + } + } + + return nil +} + +func upsertIssue(ctx context.Context, issue *issues_model.Issue) error { + sess := db.GetEngine(ctx) + + exists, err := sess.Exist(&issues_model.Issue{ID: issue.ID}) + if err != nil { + return err + } + + if !exists { + return insertIssue(ctx, issue) + } + return updateIssue(ctx, issue) +} + // InsertIssueComments inserts many comments of issues. func InsertIssueComments(comments []*issues_model.Comment) error { if len(comments) == 0 { diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 5179017afd446..4a6c5ca7f4d17 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -346,8 +346,7 @@ func (g *GiteaLocalUploader) SyncTags() error { return repo_module.SyncReleasesWithTags(g.repo, g.gitRepo) } -// CreateIssues creates issues -func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { +func (g *GiteaLocalUploader) prepareIssues(issues ...*base.Issue) ([]*issues_model.Issue, error) { iss := make([]*issues_model.Issue, 0, len(issues)) for _, issue := range issues { var labels []*issues_model.Label @@ -397,7 +396,7 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { } if err := g.remapUser(issue, &is); err != nil { - return err + return nil, err } if issue.Closed != nil { @@ -410,23 +409,32 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { CreatedUnix: timeutil.TimeStampNow(), } if err := g.remapUser(reaction, &res); err != nil { - return err + return nil, err } is.Reactions = append(is.Reactions, &res) } iss = append(iss, &is) } + return iss, nil +} - if len(iss) > 0 { - if err := models.InsertIssues(iss...); err != nil { - return err - } +// CreateIssues creates issues +func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { + iss, err := g.prepareIssues(issues...) + if err != nil { + return err + } + if len(iss) == 0 { + return nil + } - for _, is := range iss { - g.issues[is.Index] = is - } + if err := models.InsertIssues(iss...); err != nil { + return err } + for _, is := range iss { + g.issues[is.Index] = is + } return nil } @@ -836,6 +844,21 @@ func (g *GiteaLocalUploader) PatchReleases(releases ...*base.Release) error { } func (g *GiteaLocalUploader) PatchIssues(issues ...*base.Issue) error { + iss, err := g.prepareIssues(issues...) + if err != nil { + return err + } + if len(iss) == 0 { + return nil + } + + if err := models.UpsertIssues(iss...); err != nil { + return err + } + + for _, is := range iss { + g.issues[is.Index] = is + } return nil } From 0932caa18a293e3ff20230e2cb772d9c396766dc Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 12 Jul 2022 03:26:52 -0400 Subject: [PATCH 15/94] Some cleanups --- models/migrate.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/models/migrate.go b/models/migrate.go index 578ddf8bd0476..3940388a8e3bc 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -121,10 +121,6 @@ func updateIssue(ctx context.Context, issue *issues_model.Issue) error { } issueLabels := resolveIssueLabels(issue.ID, issue.Labels) if len(issueLabels) > 0 { - labelIDs := make([]int64, 0, len(issueLabels)) - for _, label := range issueLabels { - labelIDs = append(labelIDs, label.LabelID) - } // delete old labels if _, err := sess.Where("issue_id = ?", issue.ID).Delete(); err != nil { return err From 5ffadebac674f3a4dcd2beb6f7ff5bbf33271dcb Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 12 Jul 2022 03:54:01 -0400 Subject: [PATCH 16/94] Support updating pull requests --- models/migrate.go | 38 +++++++++++++++++++++++---- services/migrations/gitea_uploader.go | 31 +++++++++++++++++----- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/models/migrate.go b/models/migrate.go index 3940388a8e3bc..dd00443285197 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -107,7 +107,7 @@ func UpsertIssues(issues ...*issues_model.Issue) error { defer committer.Close() for _, issue := range issues { - if err := upsertIssue(ctx, issue); err != nil { + if _, err := upsertIssue(ctx, issue); err != nil { return err } } @@ -175,18 +175,18 @@ func updateIssue(ctx context.Context, issue *issues_model.Issue) error { return nil } -func upsertIssue(ctx context.Context, issue *issues_model.Issue) error { +func upsertIssue(ctx context.Context, issue *issues_model.Issue) (isInsert bool, err error) { sess := db.GetEngine(ctx) exists, err := sess.Exist(&issues_model.Issue{ID: issue.ID}) if err != nil { - return err + return false, err } if !exists { - return insertIssue(ctx, issue) + return true, insertIssue(ctx, issue) } - return updateIssue(ctx, issue) + return false, updateIssue(ctx, issue) } // InsertIssueComments inserts many comments of issues. @@ -250,6 +250,34 @@ func InsertPullRequests(prs ...*issues_model.PullRequest) error { return committer.Commit() } +// UpsertPullRequests inserts new pull requests and updates existing pull requests in database +func UpsertPullRequests(prs ...*issues_model.PullRequest) error { + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + sess := db.GetEngine(ctx) + for _, pr := range prs { + isInsert, err := upsertIssue(ctx, pr.Issue) + if err != nil { + return err + } + pr.IssueID = pr.Issue.ID + + if isInsert { + if _, err := sess.NoAutoTime().Insert(pr); err != nil { + return err + } + } else { + if _, err := sess.NoAutoTime().ID(pr.ID).Update(pr); err != nil { + return err + } + } + } + return committer.Commit() +} + // InsertReleases migrates release func InsertReleases(rels ...*Release) error { ctx, committer, err := db.TxContext() diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 4a6c5ca7f4d17..70868266f508b 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -488,21 +488,29 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { return models.InsertIssueComments(cms) } -// CreatePullRequests creates pull requests -func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error { +func (g *GiteaLocalUploader) preparePullRequests(prs ...*base.PullRequest) ([]*issues_model.PullRequest, error) { gprs := make([]*issues_model.PullRequest, 0, len(prs)) for _, pr := range prs { - gpr, err := g.newPullRequest(pr) + gpr, err := g.getPullRequest(pr) if err != nil { - return err + return nil, err } if err := g.remapUser(pr, gpr.Issue); err != nil { - return err + return nil, err } gprs = append(gprs, gpr) } + return gprs, nil +} + +// CreatePullRequests creates pull requests +func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error { + gprs, err := g.preparePullRequests(prs...) + if err != nil { + return err + } if err := models.InsertPullRequests(gprs...); err != nil { return err } @@ -617,7 +625,7 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head return head, nil } -func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model.PullRequest, error) { +func (g *GiteaLocalUploader) getPullRequest(pr *base.PullRequest) (*issues_model.PullRequest, error) { var labels []*issues_model.Label for _, label := range pr.Labels { lb, ok := g.labels[label.Name] @@ -867,6 +875,17 @@ func (g *GiteaLocalUploader) PatchComments(comments ...*base.Comment) error { } func (g *GiteaLocalUploader) PatchPullRequests(prs ...*base.PullRequest) error { + gprs, err := g.preparePullRequests(prs...) + if err != nil { + return err + } + if err := models.InsertPullRequests(gprs...); err != nil { + return err + } + for _, pr := range gprs { + g.issues[pr.Issue.Index] = pr.Issue + pull.AddToTaskQueue(pr) + } return nil } From 6d242993c7cfe01758f5f5698c703be445ec894b Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 12 Jul 2022 17:36:15 -0400 Subject: [PATCH 17/94] Support updating milestones --- models/migrate.go | 42 +++++++++++++++++++++++++++ services/migrations/gitea_uploader.go | 32 ++++++++++++++------ 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/models/migrate.go b/models/migrate.go index dd00443285197..6dbc36e885f40 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -39,6 +39,48 @@ func InsertMilestones(ms ...*issues_model.Milestone) (err error) { return committer.Commit() } +// UpdateMilestones updates milestones of repository. +func UpdateMilestones(ms ...*issues_model.Milestone) (err error) { + if len(ms) == 0 { + return nil + } + + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + sess := db.GetEngine(ctx) + + // check if milestone exists, if not create it + // milestones are considered as unique by RepoID and CreatedUnix + for _, m := range ms { + var existingMilestone *issues_model.Milestone + has, err := sess.Where("repo_id = ? AND created_unix = ?", m.RepoID, m.CreatedUnix).Get(&existingMilestone) + if err != nil { + return err + } + + if !has { + if _, err = sess.NoAutoTime().Insert(m); err != nil { + return err + } + } else if existingMilestone.Name != m.Name || existingMilestone.Content != m.Content || + existingMilestone.IsClosed != m.IsClosed || + existingMilestone.UpdatedUnix != m.UpdatedUnix || existingMilestone.ClosedDateUnix != m.ClosedDateUnix || + existingMilestone.DeadlineUnix != m.DeadlineUnix { + if _, err = sess.NoAutoTime().ID(existingMilestone.ID).Update(m); err != nil { + return err + } + } + } + + if _, err = sess.Exec(ctx, "UPDATE `repository` SET num_milestones = num_milestones + ? WHERE id = ?", len(ms), ms[0].RepoID); err != nil { + return err + } + return committer.Commit() +} + // InsertIssues insert issues to database func InsertIssues(issues ...*issues_model.Issue) error { ctx, committer, err := db.TxContext() diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 70868266f508b..b2a708fc190a4 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -169,8 +169,7 @@ func (g *GiteaLocalUploader) CreateTopics(topics ...string) error { return repo_model.AddTopics(g.repo.ID, topics...) } -// CreateMilestones creates milestones -func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) error { +func (g *GiteaLocalUploader) prepareMilestones(milestones ...*base.Milestone) []*issues_model.Milestone { mss := make([]*issues_model.Milestone, 0, len(milestones)) for _, milestone := range milestones { var deadline timeutil.TimeStamp @@ -195,20 +194,26 @@ func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) err } ms := issues_model.Milestone{ - RepoID: g.repo.ID, - Name: milestone.Title, - Content: milestone.Description, - IsClosed: milestone.State == "closed", - CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()), - UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()), - DeadlineUnix: deadline, + RepoID: g.repo.ID, + Name: milestone.Title, + Content: milestone.Description, + IsClosed: milestone.State == "closed", + CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()), + UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()), + ClosedDateUnix: timeutil.TimeStamp(milestone.Closed.Unix()), + DeadlineUnix: deadline, } if ms.IsClosed && milestone.Closed != nil { ms.ClosedDateUnix = timeutil.TimeStamp(milestone.Closed.Unix()) } mss = append(mss, &ms) } + return mss +} +// CreateMilestones creates milestones +func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) error { + mss := g.prepareMilestones(milestones...) err := models.InsertMilestones(mss...) if err != nil { return err @@ -833,6 +838,15 @@ func (g *GiteaLocalUploader) UpdateTopics(topics ...string) error { } func (g *GiteaLocalUploader) UpdateMilestones(milestones ...*base.Milestone) error { + mss := g.prepareMilestones(milestones...) + err := models.UpdateMilestones(mss...) + if err != nil { + return err + } + + for _, ms := range mss { + g.milestones[ms.Name] = ms.ID + } return nil } From faa0d0b76efd99640878a02ed7d23188d6f5a39d Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 12 Jul 2022 19:33:43 -0400 Subject: [PATCH 18/94] Fix milestone migrations --- services/migrations/gitea_uploader.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index b2a708fc190a4..d77be7da4ad91 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -194,14 +194,13 @@ func (g *GiteaLocalUploader) prepareMilestones(milestones ...*base.Milestone) [] } ms := issues_model.Milestone{ - RepoID: g.repo.ID, - Name: milestone.Title, - Content: milestone.Description, - IsClosed: milestone.State == "closed", - CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()), - UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()), - ClosedDateUnix: timeutil.TimeStamp(milestone.Closed.Unix()), - DeadlineUnix: deadline, + RepoID: g.repo.ID, + Name: milestone.Title, + Content: milestone.Description, + IsClosed: milestone.State == "closed", + CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()), + UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()), + DeadlineUnix: deadline, } if ms.IsClosed && milestone.Closed != nil { ms.ClosedDateUnix = timeutil.TimeStamp(milestone.Closed.Unix()) From 58958074f2e32c8f8c10bda241ad025b47f5cfd7 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 12 Jul 2022 21:36:56 -0400 Subject: [PATCH 19/94] Fix: exec sql with session --- models/migrate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrate.go b/models/migrate.go index 6dbc36e885f40..0617d86f343a7 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -33,7 +33,7 @@ func InsertMilestones(ms ...*issues_model.Milestone) (err error) { } } - if _, err = db.Exec(ctx, "UPDATE `repository` SET num_milestones = num_milestones + ? WHERE id = ?", len(ms), ms[0].RepoID); err != nil { + if _, err = sess.Exec(ctx, "UPDATE `repository` SET num_milestones = num_milestones + ? WHERE id = ?", len(ms), ms[0].RepoID); err != nil { return err } return committer.Commit() From b96fe660ba88f1d710929dc706db2f64910bb013 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 13 Jul 2022 02:51:00 -0400 Subject: [PATCH 20/94] Fix test --- models/migrate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrate.go b/models/migrate.go index 0617d86f343a7..84b979934ff2d 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -33,7 +33,7 @@ func InsertMilestones(ms ...*issues_model.Milestone) (err error) { } } - if _, err = sess.Exec(ctx, "UPDATE `repository` SET num_milestones = num_milestones + ? WHERE id = ?", len(ms), ms[0].RepoID); err != nil { + if _, err = sess.Exec("UPDATE `repository` SET num_milestones = num_milestones + ? WHERE id = ?", len(ms), ms[0].RepoID); err != nil { return err } return committer.Commit() From 8127c25de86d15f4d0a184cf43c5815884e4716a Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 13 Jul 2022 20:51:31 -0400 Subject: [PATCH 21/94] Provide preliminary implementation for releases --- models/migrate.go | 71 +++++++++++++++++++++++++++ services/migrations/gitea_uploader.go | 28 ++++++++--- 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/models/migrate.go b/models/migrate.go index 84b979934ff2d..20f9ef175ec22 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -10,6 +10,8 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/foreignreference" issues_model "code.gitea.io/gitea/models/issues" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/structs" ) @@ -348,6 +350,75 @@ func InsertReleases(rels ...*Release) error { return committer.Commit() } +// UpsertReleases inserts new releases and updates existing releases +func UpsertReleases(rels ...*Release) error { + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + sess := db.GetEngine(ctx) + + for _, rel := range rels { + exists, err := sess.Where("repo_id = ? AND tag_name = ?", rel.RepoID, rel.TagName).Exist(&Release{}) + if err != nil { + return err + } + + if !exists { + if _, err := sess.NoAutoTime().Insert(rel); err != nil { + return err + } + + if len(rel.Attachments) > 0 { + for i := range rel.Attachments { + rel.Attachments[i].ReleaseID = rel.ID + } + + if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil { + return err + } + } + } else { + if _, err := sess.NoAutoTime().Where("repo_id = ? AND tag_name = ?", rel.RepoID, rel.TagName).Update(rel); err != nil { + return err + } + + if len(rel.Attachments) > 0 { + for i := range rel.Attachments { + rel.Attachments[i].ReleaseID = rel.ID + } + + var existingReleases []*repo_model.Attachment + err := sess.Where("release_id = ?", rel.ID).Find(&existingReleases) + if err != nil { + return err + } + + if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil { + return err + } + + var ids []int64 + for _, existingRelease := range existingReleases { + // TODO: file operations are not atomic, so errors should be handled + err = storage.Attachments.Delete(existingRelease.RelativePath()) + if err != nil { + return err + } + + ids = append(ids, existingRelease.ID) + } + if _, err := sess.NoAutoTime().In("id", ids).Delete(&repo_model.Attachment{}); err != nil { + return err + } + } + } + } + + return committer.Commit() +} + // UpdateMigrationsByType updates all migrated repositories' posterid from gitServiceType to replace originalAuthorID to posterID func UpdateMigrationsByType(tp structs.GitServiceType, externalUserID string, userID int64) error { if err := issues_model.UpdateIssuesMigrationsByType(tp, externalUserID, userID); err != nil { diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 0c8f587534e1f..9acd52eea7e4f 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -248,8 +248,7 @@ func convertLabels(repoID int64, labels ...*base.Label) []*issues_model.Label { return lbs } -// CreateReleases creates releases -func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { +func (g *GiteaLocalUploader) prepareReleases(releases ...*base.Release) ([]*models.Release, error) { rels := make([]*models.Release, 0, len(releases)) for _, release := range releases { if release.Created.IsZero() { @@ -274,7 +273,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { } if err := g.remapUser(release, &rel); err != nil { - return err + return nil, err } // calc NumCommits if possible @@ -282,12 +281,12 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { commit, err := g.gitRepo.GetTagCommit(rel.TagName) if !git.IsErrNotExist(err) { if err != nil { - return fmt.Errorf("GetTagCommit[%v]: %v", rel.TagName, err) + return nil, fmt.Errorf("GetTagCommit[%v]: %v", rel.TagName, err) } rel.Sha1 = commit.ID.String() rel.NumCommits, err = commit.CommitsCount() if err != nil { - return fmt.Errorf("CommitsCount: %v", err) + return nil, fmt.Errorf("CommitsCount: %v", err) } } } @@ -332,7 +331,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { return err }() if err != nil { - return err + return nil, err } rel.Attachments = append(rel.Attachments, &attach) @@ -340,6 +339,15 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { rels = append(rels, &rel) } + return rels, nil +} + +// CreateReleases creates releases +func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { + rels, err := g.prepareReleases(releases...) + if err != nil { + return err + } return models.InsertReleases(rels...) } @@ -860,7 +868,13 @@ func (g *GiteaLocalUploader) UpdateLabels(labels ...*base.Label) error { } func (g *GiteaLocalUploader) PatchReleases(releases ...*base.Release) error { - return nil + // TODO: needs performance improvement + rels, err := g.prepareReleases(releases...) + if err != nil { + return err + } + + return models.UpsertReleases(rels...) } func (g *GiteaLocalUploader) PatchIssues(issues ...*base.Issue) error { From 07ba08000b2d51d9a88c5ff73f69dcdefbc455ca Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 14 Jul 2022 17:07:30 -0400 Subject: [PATCH 22/94] Support updating comments --- models/migrate.go | 78 +++++++++++++++++++++++++++ services/migrations/gitea_uploader.go | 28 +++++++--- 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/models/migrate.go b/models/migrate.go index 20f9ef175ec22..9a2406ef5a6be 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -274,6 +274,84 @@ func InsertIssueComments(comments []*issues_model.Comment) error { return committer.Commit() } +// UpsertIssueComments inserts many comments of issues. +func UpsertIssueComments(comments []*issues_model.Comment) error { + if len(comments) == 0 { + return nil + } + + issueIDs := make(map[int64]bool) + for _, comment := range comments { + issueIDs[comment.IssueID] = true + } + + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + + sess := db.GetEngine(ctx) + for _, comment := range comments { + exists, err := sess.Exist(&issues_model.Comment{ + IssueID: comment.IssueID, + CreatedUnix: comment.CreatedUnix, + }) + if err != nil { + return err + } + if !exists { + if _, err := sess.NoAutoTime().Insert(comment); err != nil { + return err + } + } else { + if _, err := sess.NoAutoTime().Where( + "issue_id = ? AND created_unix = ?", comment.IssueID, comment.CreatedUnix, + ).Update(comment); err != nil { + return err + } + } + + for _, reaction := range comment.Reactions { + reaction.IssueID = comment.IssueID + reaction.CommentID = comment.ID + } + if len(comment.Reactions) > 0 { + for _, reaction := range comment.Reactions { + // issue is uniquely identified by issue_id, comment_id and type + exists, err := sess.Exist(&issues_model.Reaction{ + IssueID: reaction.IssueID, + CommentID: reaction.CommentID, + Type: reaction.Type, + }) + if err != nil { + return err + } + if exists { + if _, err := sess.Where( + "issue_id = ? AND comment_id = ? AND type = ?", + reaction.IssueID, reaction.CommentID, reaction.Type, + ).Update(&reaction); err != nil { + return err + } + } else { + if _, err := sess.Insert(&reaction); err != nil { + return err + } + } + } + } + } + + for issueID := range issueIDs { + if _, err := db.Exec(ctx, "UPDATE issue set num_comments = (SELECT count(*) FROM comment WHERE issue_id = ? AND `type`=?) WHERE id = ?", + issueID, issues_model.CommentTypeComment, issueID); err != nil { + return err + } + } + return committer.Commit() +} + // InsertPullRequests inserted pull requests func InsertPullRequests(prs ...*issues_model.PullRequest) error { ctx, committer, err := db.TxContext() diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 9acd52eea7e4f..f23a8a14395ae 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -449,14 +449,13 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error { return nil } -// CreateComments creates comments of issues -func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { +func (g *GiteaLocalUploader) prepareComments(comments ...*base.Comment) ([]*issues_model.Comment, error) { cms := make([]*issues_model.Comment, 0, len(comments)) for _, comment := range comments { var issue *issues_model.Issue issue, ok := g.issues[comment.IssueIndex] if !ok { - return fmt.Errorf("comment references non existent IssueIndex %d", comment.IssueIndex) + return nil, fmt.Errorf("comment references non existent IssueIndex %d", comment.IssueIndex) } if comment.Created.IsZero() { @@ -475,7 +474,7 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { } if err := g.remapUser(comment, &cm); err != nil { - return err + return nil, err } // add reactions @@ -485,13 +484,22 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { CreatedUnix: timeutil.TimeStampNow(), } if err := g.remapUser(reaction, &res); err != nil { - return err + return nil, err } cm.Reactions = append(cm.Reactions, &res) } cms = append(cms, &cm) } + return cms, nil +} + +// CreateComments creates comments of issues +func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error { + cms, err := g.prepareComments(comments...) + if err != nil { + return err + } if len(cms) == 0 { return nil @@ -897,7 +905,15 @@ func (g *GiteaLocalUploader) PatchIssues(issues ...*base.Issue) error { } func (g *GiteaLocalUploader) PatchComments(comments ...*base.Comment) error { - return nil + cms, err := g.prepareComments(comments...) + if err != nil { + return err + } + + if len(cms) == 0 { + return nil + } + return models.UpsertIssueComments(cms) } func (g *GiteaLocalUploader) PatchPullRequests(prs ...*base.PullRequest) error { From 9892ae13e236eb5ffe112ea6c000cb0de77f1019 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Jul 2022 16:34:52 -0400 Subject: [PATCH 23/94] Add closed date to milestones --- services/migrations/gitea_uploader.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index f23a8a14395ae..926287f9d8e27 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -193,13 +193,14 @@ func (g *GiteaLocalUploader) prepareMilestones(milestones ...*base.Milestone) [] } ms := issues_model.Milestone{ - RepoID: g.repo.ID, - Name: milestone.Title, - Content: milestone.Description, - IsClosed: milestone.State == "closed", - CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()), - UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()), - DeadlineUnix: deadline, + RepoID: g.repo.ID, + Name: milestone.Title, + Content: milestone.Description, + IsClosed: milestone.State == "closed", + CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()), + UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()), + ClosedDateUnix: timeutil.TimeStamp(milestone.Closed.Unix()), + DeadlineUnix: deadline, } if ms.IsClosed && milestone.Closed != nil { ms.ClosedDateUnix = timeutil.TimeStamp(milestone.Closed.Unix()) From 97d47b22365d2d620f38751d2228ee1f5c93ddec Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Jul 2022 16:54:24 -0400 Subject: [PATCH 24/94] Fix xorm usage --- models/migrate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrate.go b/models/migrate.go index 9a2406ef5a6be..5f7877538ca68 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -166,7 +166,7 @@ func updateIssue(ctx context.Context, issue *issues_model.Issue) error { issueLabels := resolveIssueLabels(issue.ID, issue.Labels) if len(issueLabels) > 0 { // delete old labels - if _, err := sess.Where("issue_id = ?", issue.ID).Delete(); err != nil { + if _, err := sess.Table("issue_label").Where("issue_id = ?", issue.ID).Delete(); err != nil { return err } // insert new labels From 9f5721fa95598dcccd34a5915a37411dc5fd4403 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Jul 2022 17:03:29 -0400 Subject: [PATCH 25/94] Revert previous change --- services/migrations/gitea_uploader.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 926287f9d8e27..f23a8a14395ae 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -193,14 +193,13 @@ func (g *GiteaLocalUploader) prepareMilestones(milestones ...*base.Milestone) [] } ms := issues_model.Milestone{ - RepoID: g.repo.ID, - Name: milestone.Title, - Content: milestone.Description, - IsClosed: milestone.State == "closed", - CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()), - UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()), - ClosedDateUnix: timeutil.TimeStamp(milestone.Closed.Unix()), - DeadlineUnix: deadline, + RepoID: g.repo.ID, + Name: milestone.Title, + Content: milestone.Description, + IsClosed: milestone.State == "closed", + CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()), + UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()), + DeadlineUnix: deadline, } if ms.IsClosed && milestone.Closed != nil { ms.ClosedDateUnix = timeutil.TimeStamp(milestone.Closed.Unix()) From 9ac5518ee693200a0144f313a21c06aad1c4e734 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Jul 2022 17:04:16 -0400 Subject: [PATCH 26/94] Fix issue updates --- models/migrate.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/models/migrate.go b/models/migrate.go index 5f7877538ca68..4514a7e1d3b5a 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -222,7 +222,10 @@ func updateIssue(ctx context.Context, issue *issues_model.Issue) error { func upsertIssue(ctx context.Context, issue *issues_model.Issue) (isInsert bool, err error) { sess := db.GetEngine(ctx) - exists, err := sess.Exist(&issues_model.Issue{ID: issue.ID}) + exists, err := sess.Exist(&issues_model.Issue{ + RepoID: issue.RepoID, + Index: issue.Index, + }) if err != nil { return false, err } From 2ae6e4ce98d6d5ee5e525a02406925b7c5baad93 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 15 Jul 2022 20:35:39 -0400 Subject: [PATCH 27/94] Add sync repository function --- services/migrations/migrate.go | 287 +++++++++++++++++++++++++++++++++ 1 file changed, 287 insertions(+) diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index f2542173a0ee6..37f7a63c99f9e 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -12,6 +12,7 @@ import ( "net/url" "path/filepath" "strings" + "time" "code.gitea.io/gitea/models" admin_model "code.gitea.io/gitea/models/admin" @@ -462,6 +463,292 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return uploader.Finish() } +// SyncRepository syncs a repository according MigrateOptions +func SyncRepository(ctx context.Context, doer *user_model.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger, lastSynced time.Time) (*repo_model.Repository, error) { + downloader, err := newDownloader(ctx, ownerName, opts) + if err != nil { + return nil, err + } + + uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) + uploader.gitServiceType = opts.GitServiceType + + if err := syncRepository(downloader, uploader, opts, messenger, lastSynced); err != nil { + if err1 := uploader.Rollback(); err1 != nil { + log.Error("rollback failed: %v", err1) + } + if err2 := admin_model.CreateRepositoryNotice(fmt.Sprintf("Syncing repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { + log.Error("create repository notice failed: ", err2) + } + return nil, err + } + return uploader.repo, nil +} + +// syncRepository will download new information and then upload it to Uploader. +func syncRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger, lastSynced time.Time) error { + if messenger == nil { + messenger = base.NilMessenger + } + + log.Trace("syncing topics") + messenger("repo.migrate.syncing_topics") + topics, err := downloader.GetTopics() + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("syncing topics is not supported, ignored") + } + if len(topics) != 0 { + if err = uploader.UpdateTopics(topics...); err != nil { + return err + } + } + + if opts.Milestones { + log.Trace("syncing milestones") + messenger("repo.migrate.syncing_milestones") + milestones, err := downloader.GetMilestones() + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("syncing milestones is not supported, ignored") + } + + msBatchSize := uploader.MaxBatchInsertSize("milestone") + for len(milestones) > 0 { + if len(milestones) < msBatchSize { + msBatchSize = len(milestones) + } + + if err := uploader.UpdateMilestones(milestones...); err != nil { + return err + } + milestones = milestones[msBatchSize:] + } + } + + if opts.Labels { + log.Trace("syncing labels") + messenger("repo.migrate.syncing_labels") + labels, err := downloader.GetLabels() + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("syncing labels is not supported, ignored") + } + + lbBatchSize := uploader.MaxBatchInsertSize("label") + for len(labels) > 0 { + if len(labels) < lbBatchSize { + lbBatchSize = len(labels) + } + + if err := uploader.UpdateLabels(labels...); err != nil { + return err + } + labels = labels[lbBatchSize:] + } + } + + if opts.Releases { + log.Trace("syncing releases") + messenger("repo.migrate.syncing_releases") + releases, err := downloader.GetReleases() + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("syncing releases is not supported, ignored") + } + + relBatchSize := uploader.MaxBatchInsertSize("release") + for len(releases) > 0 { + if len(releases) < relBatchSize { + relBatchSize = len(releases) + } + + if err = uploader.PatchReleases(releases[:relBatchSize]...); err != nil { + return err + } + releases = releases[relBatchSize:] + } + + // Once all releases (if any) are inserted, sync any remaining non-release tags + if err = uploader.SyncTags(); err != nil { + return err + } + } + + var ( + commentBatchSize = uploader.MaxBatchInsertSize("comment") + reviewBatchSize = uploader.MaxBatchInsertSize("review") + ) + + supportAllComments := downloader.SupportGetRepoComments() + + if opts.Issues { + log.Trace("syncing issues and comments") + messenger("repo.migrate.syncing_issues") + issueBatchSize := uploader.MaxBatchInsertSize("issue") + + for i := 1; ; i++ { + issues, isEnd, err := downloader.GetNewIssues(i, issueBatchSize, lastSynced) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("syncing issues is not supported, ignored") + break + } + + if err := uploader.PatchIssues(issues...); err != nil { + return err + } + + if opts.Comments && !supportAllComments { + allComments := make([]*base.Comment, 0, commentBatchSize) + for _, issue := range issues { + log.Trace("syncing issue %d's comments", issue.Number) + comments, _, err := downloader.GetComments(issue) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("syncing comments is not supported, ignored") + } + + allComments = append(allComments, comments...) + + if len(allComments) >= commentBatchSize { + if err = uploader.PatchComments(allComments[:commentBatchSize]...); err != nil { + return err + } + + allComments = allComments[commentBatchSize:] + } + } + + if len(allComments) > 0 { + if err = uploader.PatchComments(allComments...); err != nil { + return err + } + } + } + + if isEnd { + break + } + } + } + + if opts.PullRequests { + log.Trace("syncing pull requests and comments") + messenger("repo.migrate.syncing_pulls") + prBatchSize := uploader.MaxBatchInsertSize("pullrequest") + + for i := 1; ; i++ { + prs, isEnd, err := downloader.GetNewPullRequests(i, prBatchSize, lastSynced) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("syncing pull requests is not supported, ignored") + break + } + + if err := uploader.PatchPullRequests(prs...); err != nil { + return err + } + + if opts.Comments { + if !supportAllComments { + // plain comments + allComments := make([]*base.Comment, 0, commentBatchSize) + for _, pr := range prs { + log.Trace("syncing pull request %d's comments", pr.Number) + comments, _, err := downloader.GetComments(pr) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("syncing comments is not supported, ignored") + } + + allComments = append(allComments, comments...) + + if len(allComments) >= commentBatchSize { + if err = uploader.PatchComments(allComments[:commentBatchSize]...); err != nil { + return err + } + allComments = allComments[commentBatchSize:] + } + } + if len(allComments) > 0 { + if err = uploader.PatchComments(allComments...); err != nil { + return err + } + } + } + + // migrate reviews + allReviews := make([]*base.Review, 0, reviewBatchSize) + for _, pr := range prs { + reviews, err := downloader.GetReviews(pr) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("syncing reviews is not supported, ignored") + break + } + + allReviews = append(allReviews, reviews...) + + if len(allReviews) >= reviewBatchSize { + if err = uploader.PatchReviews(allReviews[:reviewBatchSize]...); err != nil { + return err + } + allReviews = allReviews[reviewBatchSize:] + } + } + if len(allReviews) > 0 { + if err = uploader.PatchReviews(allReviews...); err != nil { + return err + } + } + } + + if isEnd { + break + } + } + } + + if opts.Comments && supportAllComments { + log.Trace("syncing comments") + for i := 1; ; i++ { + comments, isEnd, err := downloader.GetAllComments(i, commentBatchSize) + if err != nil { + return err + } + + if err := uploader.PatchComments(comments...); err != nil { + return err + } + + if isEnd { + break + } + } + } + + return uploader.Finish() +} + // Init migrations service func Init() error { // TODO: maybe we can deprecate these legacy ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS, use ALLOWED_HOST_LIST/BLOCKED_HOST_LIST instead From 06cc290657a916b70e15e518ed9e4dfb83e9cd41 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 15 Jul 2022 21:05:45 -0400 Subject: [PATCH 28/94] Get new comments/reviews --- modules/migration/downloader.go | 3 +++ modules/migration/null_downloader.go | 19 +++++++++++++++++-- services/migrations/github.go | 27 +++++++++++++++++++++++++-- services/migrations/migrate.go | 10 +++++----- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go index 02f01987cea6d..a22c737f7a709 100644 --- a/modules/migration/downloader.go +++ b/modules/migration/downloader.go @@ -30,7 +30,10 @@ type Downloader interface { // For syncing issues and pull requests GetNewIssues(page, perPage int, updatedAfter time.Time) ([]*Issue, bool, error) + GetNewComments(commentable Commentable, updatedAfter time.Time) ([]*Comment, bool, error) + GetAllNewComments(page, perPage int, updatedAfter time.Time) ([]*Comment, bool, error) GetNewPullRequests(page, perPage int, updatedAfter time.Time) ([]*PullRequest, bool, error) + GetNewReviews(reviewable Reviewable, updatedAfter time.Time) ([]*Review, error) } // DownloaderFactory defines an interface to match a downloader implementation and create a downloader diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go index be2d28923d024..3e90ff0fb24cb 100644 --- a/modules/migration/null_downloader.go +++ b/modules/migration/null_downloader.go @@ -91,10 +91,25 @@ func (n NullDownloader) SupportGetRepoComments() bool { // GetNewIssues returns new issues updated after the given time according start and limit func (n NullDownloader) GetNewIssues(page, perPage int, updatedAfter time.Time) ([]*Issue, bool, error) { - return nil, false, ErrNotSupported{Entity: "Issues"} + return nil, false, ErrNotSupported{Entity: "NewIssues"} +} + +// GetNewComments returns comments of an issue or PR after the given time +func (n NullDownloader) GetNewComments(commentable Commentable, updatedAfter time.Time) ([]*Comment, bool, error) { + return nil, false, ErrNotSupported{Entity: "NewComments"} +} + +// GetAllNewComments returns paginated comments after the given time +func (n NullDownloader) GetAllNewComments(page, perPage int, updatedAfter time.Time) ([]*Comment, bool, error) { + return nil, false, ErrNotSupported{Entity: "AllNewComments"} } // GetNewPullRequests returns pull requests after the given time according page and perPage func (n NullDownloader) GetNewPullRequests(page, perPage int, updatedAfter time.Time) ([]*PullRequest, bool, error) { - return nil, false, ErrNotSupported{Entity: "PullRequests"} + return nil, false, ErrNotSupported{Entity: "NewPullRequests"} +} + +// GetNewReviews returns new pull requests review after the given time +func (n NullDownloader) GetNewReviews(reviewable Reviewable, updatedAfter time.Time) ([]*Review, error) { + return nil, ErrNotSupported{Entity: "NewReviews"} } diff --git a/services/migrations/github.go b/services/migrations/github.go index cb218ed33e3b7..477483db00ba8 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -393,11 +393,11 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool { // GetComments returns comments according issueNumber func (g *GithubDownloaderV3) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - comments, err := g.getComments(commentable) + comments, err := g.getCommentsSince(commentable, nil) return comments, false, err } -func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.Comment, error) { +func (g *GithubDownloaderV3) getCommentsSince(commentable base.Commentable, since *time.Time) ([]*base.Comment, error) { var ( allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" @@ -406,6 +406,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. opt := &github.IssueListCommentsOptions{ Sort: &created, Direction: &asc, + Since: since, ListOptions: github.ListOptions{ PerPage: g.maxPerPage, }, @@ -466,6 +467,10 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. // GetAllComments returns repository comments according page and perPageSize func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) { + return g.getAllCommentsSince(page, perPage, nil) +} + +func (g *GithubDownloaderV3) getAllCommentsSince(page, perPage int, since *time.Time) ([]*base.Comment, bool, error) { var ( allComments = make([]*base.Comment, 0, perPage) created = "created" @@ -477,6 +482,7 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, opt := &github.IssueListCommentsOptions{ Sort: &created, Direction: &asc, + Since: since, ListOptions: github.ListOptions{ Page: page, PerPage: perPage, @@ -772,6 +778,17 @@ func (g *GithubDownloaderV3) getIssuesSince(page, perPage int, since time.Time) return allIssues, len(issues) < perPage, nil } +// GetNewComments returns comments of an issue or PR after the given time +func (g GithubDownloaderV3) GetNewComments(commentable base.Commentable, updatedAfter time.Time) ([]*base.Comment, bool, error) { + comments, err := g.getCommentsSince(commentable, &updatedAfter) + return comments, false, err +} + +// GetAllNewComments returns paginated comments after the given time +func (g GithubDownloaderV3) GetAllNewComments(page, perPage int, updatedAfter time.Time) ([]*base.Comment, bool, error) { + return g.getAllCommentsSince(page, perPage, &updatedAfter) +} + // GetNewPullRequests returns pull requests after the given time according page and perPage func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter time.Time) ([]*base.PullRequest, bool, error) { // Every pull request is an issue, and only Issues API provides parameter `since`, @@ -818,6 +835,12 @@ func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter return allPRs, len(issues) < perPage, nil } +// GetNewReviews returns new pull requests review after the given time +func (g GithubDownloaderV3) GetNewReviews(reviewable base.Reviewable, updatedAfter time.Time) ([]*base.Review, error) { + // Github does not support since parameter for reviews, so we need to get all reviews + return g.GetReviews(reviewable) +} + func (g *GithubDownloaderV3) convertGithubPullRequest(pr *github.PullRequest, perPage int) (*base.PullRequest, error) { labels := make([]*base.Label, 0, len(pr.Labels)) for _, l := range pr.Labels { diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 37f7a63c99f9e..ca56344b7aeae 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -613,7 +613,7 @@ func syncRepository(downloader base.Downloader, uploader base.Uploader, opts bas allComments := make([]*base.Comment, 0, commentBatchSize) for _, issue := range issues { log.Trace("syncing issue %d's comments", issue.Number) - comments, _, err := downloader.GetComments(issue) + comments, _, err := downloader.GetNewComments(issue, lastSynced) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -670,7 +670,7 @@ func syncRepository(downloader base.Downloader, uploader base.Uploader, opts bas allComments := make([]*base.Comment, 0, commentBatchSize) for _, pr := range prs { log.Trace("syncing pull request %d's comments", pr.Number) - comments, _, err := downloader.GetComments(pr) + comments, _, err := downloader.GetNewComments(pr, lastSynced) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -694,10 +694,10 @@ func syncRepository(downloader base.Downloader, uploader base.Uploader, opts bas } } - // migrate reviews + // sync reviews allReviews := make([]*base.Review, 0, reviewBatchSize) for _, pr := range prs { - reviews, err := downloader.GetReviews(pr) + reviews, err := downloader.GetNewReviews(pr, lastSynced) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -731,7 +731,7 @@ func syncRepository(downloader base.Downloader, uploader base.Uploader, opts bas if opts.Comments && supportAllComments { log.Trace("syncing comments") for i := 1; ; i++ { - comments, isEnd, err := downloader.GetAllComments(i, commentBatchSize) + comments, isEnd, err := downloader.GetAllNewComments(i, commentBatchSize, lastSynced) if err != nil { return err } From 5b1c73ecacc8655351d7654b9d983444e4987050 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 15 Jul 2022 23:33:55 -0400 Subject: [PATCH 29/94] Allow syncing with mirror option checked --- modules/migration/downloader.go | 1 + modules/migration/null_downloader.go | 5 +++++ routers/web/repo/migrate.go | 8 -------- services/migrations/migrate.go | 5 +++++ 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go index a22c737f7a709..9e1c55eed9612 100644 --- a/modules/migration/downloader.go +++ b/modules/migration/downloader.go @@ -29,6 +29,7 @@ type Downloader interface { FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) // For syncing issues and pull requests + SupportSyncing() bool GetNewIssues(page, perPage int, updatedAfter time.Time) ([]*Issue, bool, error) GetNewComments(commentable Commentable, updatedAfter time.Time) ([]*Comment, bool, error) GetAllNewComments(page, perPage int, updatedAfter time.Time) ([]*Comment, bool, error) diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go index 3e90ff0fb24cb..b97b122243870 100644 --- a/modules/migration/null_downloader.go +++ b/modules/migration/null_downloader.go @@ -89,6 +89,11 @@ func (n NullDownloader) SupportGetRepoComments() bool { return false } +// SupportSyncing returns true if it supports syncing issues/PRs/etc from pull mirror +func (n NullDownloader) SupportSyncing() bool { + return false +} + // GetNewIssues returns new issues updated after the given time according start and limit func (n NullDownloader) GetNewIssues(page, perPage int, updatedAfter time.Time) ([]*Issue, bool, error) { return nil, false, ErrNotSupported{Entity: "NewIssues"} diff --git a/routers/web/repo/migrate.go b/routers/web/repo/migrate.go index 393f8ed3d9316..76b2a857fb6f6 100644 --- a/routers/web/repo/migrate.go +++ b/routers/web/repo/migrate.go @@ -223,14 +223,6 @@ func MigratePost(ctx *context.Context) { PullRequests: form.PullRequests, Releases: form.Releases, } - if opts.Mirror { - opts.Issues = false - opts.Milestones = false - opts.Labels = false - opts.Comments = false - opts.PullRequests = false - opts.Releases = false - } err = repo_model.CheckCreateRepository(ctx.Doer, ctxUser, opts.RepoName, false) if err != nil { diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index ca56344b7aeae..8126cae3d1ca6 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -470,6 +470,11 @@ func SyncRepository(ctx context.Context, doer *user_model.User, ownerName string return nil, err } + if !downloader.SupportSyncing() { + log.Info("repository syncing is not supported, ignored") + return nil, nil + } + uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) uploader.gitServiceType = opts.GitServiceType From f627311ad7c7e4d8b9cdba338c95b1bccb6b2973 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 16 Jul 2022 04:36:03 -0400 Subject: [PATCH 30/94] Display pull request for mirror repo --- models/repo/repo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/repo/repo.go b/models/repo/repo.go index bb2a7468ff42a..0d73e2a813ac1 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -516,12 +516,12 @@ func (repo *Repository) CanCreateBranch() bool { // CanEnablePulls returns true if repository meets the requirements of accepting pulls. func (repo *Repository) CanEnablePulls() bool { - return !repo.IsMirror && !repo.IsEmpty + return !repo.IsEmpty } // AllowsPulls returns true if repository meets the requirements of accepting pulls and has them enabled. func (repo *Repository) AllowsPulls() bool { - return repo.CanEnablePulls() && repo.UnitEnabled(unit.TypePullRequests) + return repo.CanEnablePulls() && !repo.IsMirror && repo.UnitEnabled(unit.TypePullRequests) } // CanEnableEditor returns true if repository meets the requirements of web editor. From 9ee3ba59ce38ae05f3cb261fb4dcbe7e22550fd3 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 16 Jul 2022 05:00:34 -0400 Subject: [PATCH 31/94] Disable issue on mirror repo --- models/repo/repo.go | 6 ++++++ modules/context/repo.go | 1 + templates/repo/issue/list.tmpl | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/models/repo/repo.go b/models/repo/repo.go index 0d73e2a813ac1..0d8b8dfcfaf90 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -524,6 +524,12 @@ func (repo *Repository) AllowsPulls() bool { return repo.CanEnablePulls() && !repo.IsMirror && repo.UnitEnabled(unit.TypePullRequests) } +// AllowsIssues returns true if repository meets the requirements of accepting issues and has them enabled. +func (repo *Repository) AllowsIssues() bool { + // TODO: disable only when issues are synced + return !repo.IsMirror && repo.UnitEnabled(unit.TypePullRequests) +} + // CanEnableEditor returns true if repository meets the requirements of web editor. func (repo *Repository) CanEnableEditor() bool { return !repo.IsMirror diff --git a/modules/context/repo.go b/modules/context/repo.go index 183637391862e..b946d48b7b062 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -709,6 +709,7 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) { } ctx.Data["CanCompareOrPull"] = canCompare ctx.Data["PullRequestCtx"] = ctx.Repo.PullRequest + ctx.Data["AllowsIssues"] = repo.AllowsIssues() if ctx.Repo.Repository.Status == repo_model.RepositoryPendingTransfer { repoTransfer, err := models.GetPendingRepositoryTransfer(ctx.Repo.Repository) diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index 04f7dcd6ae0d8..a4a60b3d15b4c 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -12,7 +12,7 @@ {{if not .Repository.IsArchived}}
{{if .PageIsIssueList}} - {{.locale.Tr "repo.issues.new"}} + {{.locale.Tr "repo.issues.new"}} {{else}} {{.locale.Tr "repo.pulls.new"}} {{end}} From bbd464857c785d24116b473c3743e12da0c7b8be Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 22 Jul 2022 05:09:05 -0400 Subject: [PATCH 32/94] Add sync items to Mirror struct --- models/repo/mirror.go | 8 ++++++++ modules/repository/repo.go | 17 ++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/models/repo/mirror.go b/models/repo/mirror.go index 8f96e8cee1851..01442e7503ee1 100644 --- a/models/repo/mirror.go +++ b/models/repo/mirror.go @@ -27,6 +27,14 @@ type Mirror struct { Interval time.Duration EnablePrune bool `xorm:"NOT NULL DEFAULT true"` + SyncWiki bool + SyncIssues bool + SyncMilestones bool + SyncLabels bool + SyncReleases bool + SyncComments bool + SyncPullRequests bool + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 436045146a99a..0f273ea97e8a6 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -177,11 +177,18 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, if opts.Mirror { mirrorModel := repo_model.Mirror{ - RepoID: repo.ID, - Interval: setting.Mirror.DefaultInterval, - EnablePrune: true, - NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), - LFS: opts.LFS, + RepoID: repo.ID, + Interval: setting.Mirror.DefaultInterval, + SyncWiki: opts.Wiki, + SyncIssues: opts.Issues, + SyncMilestones: opts.Milestones, + SyncLabels: opts.Labels, + SyncReleases: opts.Releases, + SyncComments: opts.Comments, + SyncPullRequests: opts.PullRequests, + EnablePrune: true, + NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), + LFS: opts.LFS, } if opts.LFS { mirrorModel.LFSEndpoint = opts.LFSEndpoint From a0303a286a0301ddea4555f86aded72998d0f996 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 22 Jul 2022 05:37:31 -0400 Subject: [PATCH 33/94] Support syncing reviews --- models/issues/review.go | 118 +++++++++++++++++++++++--- services/migrations/gitea_uploader.go | 28 ++++-- 2 files changed, 128 insertions(+), 18 deletions(-) diff --git a/models/issues/review.go b/models/issues/review.go index 5835900801778..ba12876dc54f5 100644 --- a/models/issues/review.go +++ b/models/issues/review.go @@ -626,17 +626,7 @@ func InsertReviews(reviews []*Review) error { return err } - if _, err := sess.NoAutoTime().Insert(&Comment{ - Type: CommentTypeReview, - Content: review.Content, - PosterID: review.ReviewerID, - OriginalAuthor: review.OriginalAuthor, - OriginalAuthorID: review.OriginalAuthorID, - IssueID: review.IssueID, - ReviewID: review.ID, - CreatedUnix: review.CreatedUnix, - UpdatedUnix: review.UpdatedUnix, - }); err != nil { + if _, err := sess.NoAutoTime().Insert(generateCommentFromReview(review)); err != nil { return err } @@ -654,6 +644,112 @@ func InsertReviews(reviews []*Review) error { return committer.Commit() } +// UpsertReviews inserts new reviews and updates existing ones. +// This function is used for syncing from the pull mirror. +func UpsertReviews(reviews []*Review) error { + ctx, committer, err := db.TxContext() + if err != nil { + return err + } + defer committer.Close() + sess := db.GetEngine(ctx) + + for _, review := range reviews { + exists, err := sess.Where("issue_id = ? AND created_unix", review.IssueID, review.CreatedUnix).Exist(&Review{}) + if err != nil { + return err + } + + if !exists { + if _, err := sess.NoAutoTime().Insert(review); err != nil { + return err + } + + if _, err := sess.NoAutoTime().Insert(generateCommentFromReview(review)); err != nil { + return err + } + + for _, c := range review.Comments { + c.ReviewID = review.ID + } + + if len(review.Comments) > 0 { + if _, err := sess.NoAutoTime().Insert(review.Comments); err != nil { + return err + } + } + } else { + if _, err = sess.NoAutoTime().Where("issue_id = ? AND created_unix", review.IssueID, review.CreatedUnix).Update(review); err != nil { + return err + } + + // Get id of the review + if err = sess.NoAutoTime().Where("issue_id = ? AND created_unix", review.IssueID, review.CreatedUnix).Find(review); err != nil { + return err + } + + comment := generateCommentFromReview(review) + exists, err := existsCommentByReviewIDAndCreatedUnix(sess, comment) + if err != nil { + return err + } + + if !exists { + if _, err := sess.NoAutoTime().Insert(comment); err != nil { + return err + } + } else { + if _, err := sess.NoAutoTime().Where("review_id = ? AND created_unix = ?", review.ID, comment.CreatedUnix).Update(comment); err != nil { + return err + } + } + + for _, c := range review.Comments { + c.ReviewID = review.ID + } + + if len(review.Comments) > 0 { + for _, comment := range review.Comments { + exists, err := existsCommentByReviewIDAndCreatedUnix(sess, comment) + if err != nil { + return err + } + + if !exists { + if _, err := sess.NoAutoTime().Insert(comment); err != nil { + return err + } + } else { + if _, err := sess.NoAutoTime().Where("review_id = ? AND created_unix = ?", review.ID, comment.CreatedUnix).Update(comment); err != nil { + return err + } + } + } + } + } + } + + return committer.Commit() +} + +func existsCommentByReviewIDAndCreatedUnix(sess db.Engine, comment *Comment) (bool, error) { + return sess.Where("review_id = ? AND created_unix = ?", comment.ReviewID, comment.CreatedUnix).Exist(&Comment{}) +} + +func generateCommentFromReview(review *Review) *Comment { + return &Comment{ + Type: CommentTypeReview, + Content: review.Content, + PosterID: review.ReviewerID, + OriginalAuthor: review.OriginalAuthor, + OriginalAuthorID: review.OriginalAuthorID, + IssueID: review.IssueID, + ReviewID: review.ID, + CreatedUnix: review.CreatedUnix, + UpdatedUnix: review.UpdatedUnix, + } +} + // AddReviewRequest add a review request from one reviewer func AddReviewRequest(issue *Issue, reviewer, doer *user_model.User) (*Comment, error) { ctx, committer, err := db.TxContext() diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index f23a8a14395ae..559240a80a051 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -747,14 +747,13 @@ func convertReviewState(state string) issues_model.ReviewType { } } -// CreateReviews create pull request reviews of currently migrated issues -func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { +func (g *GiteaLocalUploader) prepareReviews(reviews ...*base.Review) ([]*issues_model.Review, error) { cms := make([]*issues_model.Review, 0, len(reviews)) for _, review := range reviews { var issue *issues_model.Issue issue, ok := g.issues[review.IssueIndex] if !ok { - return fmt.Errorf("review references non existent IssueIndex %d", review.IssueIndex) + return nil, fmt.Errorf("review references non existent IssueIndex %d", review.IssueIndex) } if review.CreatedAt.IsZero() { review.CreatedAt = time.Unix(int64(issue.CreatedUnix), 0) @@ -770,7 +769,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { } if err := g.remapUser(review, &cm); err != nil { - return err + return nil, err } // get pr @@ -779,7 +778,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { var err error pr, err = issues_model.GetPullRequestByIssueIDWithNoAttributes(issue.ID) if err != nil { - return err + return nil, err } g.prCache[issue.ID] = pr } @@ -833,7 +832,7 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { } if err := g.remapUser(review, &c); err != nil { - return err + return nil, err } cm.Comments = append(cm.Comments, &c) @@ -842,6 +841,16 @@ func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { cms = append(cms, &cm) } + return cms, nil +} + +// CreateReviews create pull request reviews of currently migrated issues +func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error { + cms, err := g.prepareReviews(reviews...) + if err != nil { + return err + } + return issues_model.InsertReviews(cms) } @@ -932,7 +941,12 @@ func (g *GiteaLocalUploader) PatchPullRequests(prs ...*base.PullRequest) error { } func (g *GiteaLocalUploader) PatchReviews(reviews ...*base.Review) error { - return nil + cms, err := g.prepareReviews(reviews...) + if err != nil { + return err + } + + return issues_model.UpsertReviews(cms) } // Rollback when migrating failed, this will rollback all the changes. From 73a5a7c3dd8f1f59b3d85b059a5968109891e633 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Thu, 6 Apr 2023 02:34:30 -0400 Subject: [PATCH 34/94] Fix lint issue caused by merge --- models/repo/topic.go | 2 +- services/migrations/migrate.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/repo/topic.go b/models/repo/topic.go index e5a4ab8f5ddbf..967cd64b62044 100644 --- a/models/repo/topic.go +++ b/models/repo/topic.go @@ -268,7 +268,7 @@ func AddTopic(repoID int64, topicName string) (*Topic, error) { } func AddTopics(repoID int64, topicNames ...string) error { - ctx, committer, err := db.TxContext() + ctx, committer, err := db.TxContext(db.DefaultContext) if err != nil { return err } diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 1457cae93cb5b..4c92e20fbb3d0 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -502,7 +502,7 @@ func SyncRepository(ctx context.Context, doer *user_model.User, ownerName string if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) } - if err2 := admin_model.CreateRepositoryNotice(fmt.Sprintf("Syncing repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { + if err2 := system_model.CreateRepositoryNotice(fmt.Sprintf("Syncing repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { log.Error("create repository notice failed: ", err2) } return nil, err From 12851f2f79af3b2b6a7c72d8e81561519f4ed2dc Mon Sep 17 00:00:00 2001 From: harryzcy Date: Thu, 6 Apr 2023 02:39:14 -0400 Subject: [PATCH 35/94] Avoid duplicate code --- services/migrations/github.go | 76 +---------------------------------- 1 file changed, 1 insertion(+), 75 deletions(-) diff --git a/services/migrations/github.go b/services/migrations/github.go index 83136be4bf948..c43847ccac4cd 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -576,81 +576,7 @@ func (g *GithubDownloaderV3) getAllCommentsSince(page, perPage int, since *time. // GetPullRequests returns pull requests according page and perPage func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { - if perPage > g.maxPerPage { - perPage = g.maxPerPage - } - opt := &github.PullRequestListOptions{ - Sort: "created", - Direction: "asc", - State: "all", - ListOptions: github.ListOptions{ - PerPage: perPage, - Page: page, - }, - } - allPRs := make([]*base.PullRequest, 0, perPage) - g.waitAndPickClient() - prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) - if err != nil { - return nil, false, fmt.Errorf("error while listing repos: %w", err) - } - log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs)) - g.setRate(&resp.Rate) - for _, pr := range prs { - labels := make([]*base.Label, 0, len(pr.Labels)) - for _, l := range pr.Labels { - labels = append(labels, convertGithubLabel(l)) - } - - // get reactions - reactions, err := g.getIssueReactions(pr.GetNumber(), perPage) - if err != nil { - return nil, false, err - } - - // download patch and saved as tmp file - g.waitAndPickClient() - - allPRs = append(allPRs, &base.PullRequest{ - Title: pr.GetTitle(), - Number: int64(pr.GetNumber()), - PosterID: pr.GetUser().GetID(), - PosterName: pr.GetUser().GetLogin(), - PosterEmail: pr.GetUser().GetEmail(), - Content: pr.GetBody(), - Milestone: pr.GetMilestone().GetTitle(), - State: pr.GetState(), - Created: pr.GetCreatedAt(), - Updated: pr.GetUpdatedAt(), - Closed: pr.ClosedAt, - Labels: labels, - Merged: pr.MergedAt != nil, - MergeCommitSHA: pr.GetMergeCommitSHA(), - MergedTime: pr.MergedAt, - IsLocked: pr.ActiveLockReason != nil, - Head: base.PullRequestBranch{ - Ref: pr.GetHead().GetRef(), - SHA: pr.GetHead().GetSHA(), - OwnerName: pr.GetHead().GetUser().GetLogin(), - RepoName: pr.GetHead().GetRepo().GetName(), - CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here - }, - Base: base.PullRequestBranch{ - Ref: pr.GetBase().GetRef(), - SHA: pr.GetBase().GetSHA(), - RepoName: pr.GetBase().GetRepo().GetName(), - OwnerName: pr.GetBase().GetUser().GetLogin(), - }, - PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here - Reactions: reactions, - ForeignIndex: int64(*pr.Number), - }) - - // SECURITY: Ensure that the PR is safe - _ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g) - } - - return allPRs, len(prs) < perPage, nil + return g.GetNewPullRequests(page, perPage, time.Time{}) } func convertGithubReview(r *github.PullRequestReview) *base.Review { From e3a16ff8d1a6cfba2ced4b130743c54482145ab7 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Thu, 6 Apr 2023 02:45:50 -0400 Subject: [PATCH 36/94] Fix SPDX-License-Identifier --- modules/migration/null_uploader.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/migration/null_uploader.go b/modules/migration/null_uploader.go index 97ceceba9fd30..52130c9c3d070 100644 --- a/modules/migration/null_uploader.go +++ b/modules/migration/null_uploader.go @@ -1,6 +1,5 @@ -// Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT package migration From 1c8b8ef8a06b739ba8aef6dcc1202574d459d021 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Thu, 6 Apr 2023 04:09:04 -0400 Subject: [PATCH 37/94] Add comments --- services/migrations/github.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/services/migrations/github.go b/services/migrations/github.go index c43847ccac4cd..7dc89f2b5b1ff 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -502,6 +502,8 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, return g.getAllCommentsSince(page, perPage, nil) } +// GetAllCommentsSince returns repository comments since a time. +// If since is nil, it will return all comments. func (g *GithubDownloaderV3) getAllCommentsSince(page, perPage int, since *time.Time) ([]*base.Comment, bool, error) { var ( allComments = make([]*base.Comment, 0, perPage) @@ -579,7 +581,8 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq return g.GetNewPullRequests(page, perPage, time.Time{}) } -func convertGithubReview(r *github.PullRequestReview) *base.Review { +// convertGithubReview converts github review to Gitea review +func (g *GithubDownloaderV3) convertGithubReview(r *github.PullRequestReview) *base.Review { return &base.Review{ ID: r.GetID(), ReviewerID: r.GetUser().GetID(), @@ -655,7 +658,7 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev } g.setRate(&resp.Rate) for _, review := range reviews { - r := convertGithubReview(review) + r := g.convertGithubReview(review) r.IssueIndex = reviewable.GetLocalIndex() // retrieve all review comments opt2 := &github.ListOptions{ @@ -717,7 +720,8 @@ func (g *GithubDownloaderV3) GetNewIssues(page, perPage int, updatedAfter time.T return g.getIssuesSince(page, perPage, updatedAfter) } -// getIssuesSince returns issues given page, perPage and since +// getIssuesSince returns issues given page, perPage and since. +// when since is empty, it will return all issues func (g *GithubDownloaderV3) getIssuesSince(page, perPage int, since time.Time) ([]*base.Issue, bool, error) { if perPage > g.maxPerPage { perPage = g.maxPerPage @@ -899,6 +903,7 @@ func (g *GithubDownloaderV3) convertGithubPullRequest(pr *github.PullRequest, pe }, nil } +// getIssueReactions returns reactions using Github API func (g *GithubDownloaderV3) getIssueReactions(number, perPage int) ([]*base.Reaction, error) { var reactions []*base.Reaction if !g.SkipReactions { From 9af62926bfce56d998949321c6370b9ea52537ca Mon Sep 17 00:00:00 2001 From: harryzcy Date: Thu, 6 Apr 2023 12:09:22 -0400 Subject: [PATCH 38/94] Fix error when syncing comments when PR is not synced --- services/migrations/gitea_uploader.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 44070627b778d..19dcfae9354fd 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -476,7 +476,10 @@ func (g *GiteaLocalUploader) prepareComments(comments ...*base.Comment) ([]*issu var issue *issues_model.Issue issue, ok := g.issues[comment.IssueIndex] if !ok { - return nil, fmt.Errorf("comment references non existent IssueIndex %d", comment.IssueIndex) + // ignore comments for non existent issues + // It can happen when a comment belongs to a pull request, but the pull request is not imported + log.Warn("Ignoring comment for non existent issue %d", comment.IssueIndex) + continue } if comment.Created.IsZero() { From acd0ad220fcf36b7a752216a14a6a5bd88c1e7a1 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Thu, 6 Apr 2023 12:35:58 -0400 Subject: [PATCH 39/94] CheckAndEnsureSafePR was ignored when merging --- services/migrations/github.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/migrations/github.go b/services/migrations/github.go index 7dc89f2b5b1ff..040e8be662e46 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -841,6 +841,9 @@ func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter return nil, false, err } allPRs = append(allPRs, basePR) + + // SECURITY: Ensure that the PR is safe + _ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g) } return allPRs, len(issues) < perPage, nil From 9887d8d867e17331d241eb99cf65ec2741c3b640 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 8 Apr 2023 13:11:04 -0400 Subject: [PATCH 40/94] Fix download pr tests --- services/migrations/github.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/services/migrations/github.go b/services/migrations/github.go index b8f000de43119..5732cea3866ce 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -807,11 +807,9 @@ func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter if perPage > g.maxPerPage { perPage = g.maxPerPage } - opt := &github.IssueListByRepoOptions{ - Sort: "created", - Direction: "asc", - State: "all", - Since: updatedAfter, + opt := &github.SearchOptions{ + Sort: "created", + Order: "asc", ListOptions: github.ListOptions{ PerPage: perPage, Page: page, @@ -820,17 +818,22 @@ func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter allPRs := make([]*base.PullRequest, 0, perPage) g.waitAndPickClient() - issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt) + + searchQuery := fmt.Sprintf("repo:%s/%s is:pr", g.repoOwner, g.repoName) + if !updatedAfter.IsZero() { + // timezone denoted by plus, rather than 'Z', + // according to https://docs.github.com/en/search-github/searching-on-github/searching-issues-and-pull-requests#search-by-when-an-issue-or-pull-request-was-created-or-last-updated + timeStr := updatedAfter.Format("2006-01-02T15:04:05+07:00") + searchQuery += fmt.Sprintf(" updated:>=%s", timeStr) + } + + result, resp, err := g.getClient().Search.Issues(g.ctx, searchQuery, opt) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } - log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues)) + log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(result.Issues)) g.setRate(&resp.Rate) - for _, issue := range issues { - if !issue.IsPullRequest() { - continue - } - + for _, issue := range result.Issues { pr, resp, err := g.getClient().PullRequests.Get(g.ctx, g.repoOwner, g.repoName, issue.GetNumber()) if err != nil { return nil, false, fmt.Errorf("error while getting repo pull request: %v", err) @@ -846,7 +849,7 @@ func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter _ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g) } - return allPRs, len(issues) < perPage, nil + return allPRs, len(result.Issues) < perPage, nil } // GetNewReviews returns new pull requests review after the given time From 8481bcf623ac3c2fc38cff8e54fae19bba142acb Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 8 Apr 2023 13:24:35 -0400 Subject: [PATCH 41/94] Correctly format ISO 8601 time --- services/migrations/github.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/services/migrations/github.go b/services/migrations/github.go index 5732cea3866ce..a696ee8a34c90 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -802,8 +802,10 @@ func (g GithubDownloaderV3) GetAllNewComments(page, perPage int, updatedAfter ti // GetNewPullRequests returns pull requests after the given time according page and perPage func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter time.Time) ([]*base.PullRequest, bool, error) { - // Every pull request is an issue, and only Issues API provides parameter `since`, - // So we should get issues IDs first and then get pull requests + // Pulls API doesn't have parameter `since`, so we have to use Search API instead. + // By specifying `repo:owner/repo is:pr` in the query, we can get all pull requests of the repository. + // In addition, we can specify `updated:>=YYYY-MM-DDTHH:MM:SS+00:00` to get pull requests updated after the given time. + if perPage > g.maxPerPage { perPage = g.maxPerPage } @@ -823,7 +825,7 @@ func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter if !updatedAfter.IsZero() { // timezone denoted by plus, rather than 'Z', // according to https://docs.github.com/en/search-github/searching-on-github/searching-issues-and-pull-requests#search-by-when-an-issue-or-pull-request-was-created-or-last-updated - timeStr := updatedAfter.Format("2006-01-02T15:04:05+07:00") + timeStr := updatedAfter.Format("2006-01-02T15:04:05-07:00") searchQuery += fmt.Sprintf(" updated:>=%s", timeStr) } From 448ccfd74f8708bd8e5e725978b68d2cefd5af5b Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 8 Apr 2023 13:28:11 -0400 Subject: [PATCH 42/94] Update comments --- services/migrations/github.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/migrations/github.go b/services/migrations/github.go index a696ee8a34c90..5f562384397ac 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -801,6 +801,7 @@ func (g GithubDownloaderV3) GetAllNewComments(page, perPage int, updatedAfter ti } // GetNewPullRequests returns pull requests after the given time according page and perPage +// If `updatedAfter` is zero-valued, it will return all pull requests func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter time.Time) ([]*base.PullRequest, bool, error) { // Pulls API doesn't have parameter `since`, so we have to use Search API instead. // By specifying `repo:owner/repo is:pr` in the query, we can get all pull requests of the repository. @@ -823,7 +824,8 @@ func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter searchQuery := fmt.Sprintf("repo:%s/%s is:pr", g.repoOwner, g.repoName) if !updatedAfter.IsZero() { - // timezone denoted by plus, rather than 'Z', + // GitHub requires time to be later than 1970-01-01, so we should skip `updated` part if it's zero. + // Timezone is denoted by plus/minus UTC offset, rather than 'Z', // according to https://docs.github.com/en/search-github/searching-on-github/searching-issues-and-pull-requests#search-by-when-an-issue-or-pull-request-was-created-or-last-updated timeStr := updatedAfter.Format("2006-01-02T15:04:05-07:00") searchQuery += fmt.Sprintf(" updated:>=%s", timeStr) From 786d15f77cddc29a1521fbe8d9692a0e6b9b3fd5 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 8 Apr 2023 23:00:03 -0400 Subject: [PATCH 43/94] Fix a sql query --- models/issues/review.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/issues/review.go b/models/issues/review.go index b4047d4fb2247..4f8d2c70a0114 100644 --- a/models/issues/review.go +++ b/models/issues/review.go @@ -681,12 +681,12 @@ func UpsertReviews(reviews []*Review) error { } } } else { - if _, err = sess.NoAutoTime().Where("issue_id = ? AND created_unix", review.IssueID, review.CreatedUnix).Update(review); err != nil { + if _, err = sess.NoAutoTime().Where("issue_id = ? AND created_unix = ?", review.IssueID, review.CreatedUnix).Update(review); err != nil { return err } // Get id of the review - if err = sess.NoAutoTime().Where("issue_id = ? AND created_unix", review.IssueID, review.CreatedUnix).Find(review); err != nil { + if err = sess.NoAutoTime().Where("issue_id = ? AND created_unix = ?", review.IssueID, review.CreatedUnix).Find(review); err != nil { return err } From e50ab380763e9ba4a1536643239d3d50dcf43c33 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Wed, 3 May 2023 20:30:45 -0400 Subject: [PATCH 44/94] Fix lint --- templates/repo/issue/list.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/issue/list.tmpl b/templates/repo/issue/list.tmpl index 9af79a140a033..f857ef8144fec 100644 --- a/templates/repo/issue/list.tmpl +++ b/templates/repo/issue/list.tmpl @@ -317,4 +317,4 @@ {{template "shared/issuelist" dict "." . "listType" "repo"}}
-{{template "base/footer" .}} \ No newline at end of file +{{template "base/footer" .}} From 15d16a56076e0cf44a7f312ae6556be17e6d07d3 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 6 May 2023 01:24:40 -0400 Subject: [PATCH 45/94] Identify the right place to do continuous sync --- services/mirror/mirror_pull.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 60699294c18bd..312869164ea25 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -308,7 +308,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo log.Error("SyncMirrors [repo: %-v]: failed to update size for mirror repository: %v", m.Repo, err) } - if m.Repo.HasWiki() { + if m.Repo.HasWiki() && m.SyncWiki { log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo) stderrBuilder.Reset() stdoutBuilder.Reset() @@ -372,6 +372,8 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo) } + // TODO：sync issues, prs, etc. + log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo) branches, _, err := git.GetBranchesByPath(ctx, m.Repo.RepoPath(), 0, 0) if err != nil { From ce51877f7058f4545262734ecb12944d07bde04c Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 6 May 2023 16:51:09 -0400 Subject: [PATCH 46/94] Sync mirror after migration --- services/mirror/mirror_pull.go | 66 ++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 312869164ea25..4d7cd03b2c745 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/proxy" @@ -23,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/services/migrations" ) // gitShortEmptySha Git short empty SHA @@ -196,8 +198,8 @@ func pruneBrokenReferences(ctx context.Context, return pruneErr } -// runSync returns true if sync finished without error. -func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) { +// runSyncGit returns true if sync git repos finished without error. +func runSyncGit(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) { repoPath := m.Repo.RepoPath() wikiPath := m.Repo.WikiPath() timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second @@ -372,8 +374,6 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo) } - // TODO：sync issues, prs, etc. - log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo) branches, _, err := git.GetBranchesByPath(ctx, m.Repo.RepoPath(), 0, 0) if err != nil { @@ -389,6 +389,53 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo return parseRemoteUpdateOutput(output), true } +// runSyncMisc runs the sync of Issues, Pull Requests, Reviews, Topics, Releases, Labels, and Milestones. +// It returns true if the sync was successful. +func runSyncMisc(ctx context.Context, m *repo_model.Mirror) bool { + repo := m.GetRepository() + + remoteURL, remoteErr := git.GetRemoteURL(ctx, repo.RepoPath(), m.GetRemoteName()) + if remoteErr != nil { + log.Error("SyncMirrors [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr) + return false + } + + password, ok := remoteURL.User.Password() + if !ok { + password = "" + } + + opts := migration.MigrateOptions{ + CloneAddr: repo.OriginalURL, + AuthUsername: remoteURL.User.Username(), + AuthPassword: password, + UID: int(repo.OwnerID), + RepoName: repo.Name, + Mirror: true, + LFS: m.LFS, + LFSEndpoint: m.LFSEndpoint, + GitServiceType: repo.OriginalServiceType, + Wiki: repo.HasWiki() && m.SyncWiki, + Issues: m.SyncIssues, + Milestones: m.SyncMilestones, + Labels: m.SyncLabels, + Releases: m.SyncReleases, + Comments: m.SyncComments, + PullRequests: m.SyncPullRequests, + // Topics + // ReleaseAssets + MigrateToRepoID: repo.ID, + MirrorInterval: m.Interval.String(), + } + _, err := migrations.SyncRepository(ctx, repo.Owner, repo.OwnerName, opts, nil, m.UpdatedUnix.AsTime()) + if err != nil { + log.Error("SyncMirrors [repo: %-v]: failed to sync repository: %v", m.Repo, err) + return false + } + + return true +} + // SyncPullMirror starts the sync of the pull mirror and schedules the next run. func SyncPullMirror(ctx context.Context, repoID int64) bool { log.Trace("SyncMirrors [repo_id: %v]", repoID) @@ -411,8 +458,8 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Syncing Mirror %s/%s", m.Repo.OwnerName, m.Repo.Name)) defer finished() - log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo) - results, ok := runSync(ctx, m) + log.Trace("SyncMirrors [repo: %-v]: Running Sync Git", m.Repo) + results, ok := runSyncGit(ctx, m) if !ok { if err = repo_model.TouchMirror(ctx, m); err != nil { log.Error("SyncMirrors [repo: %-v]: failed to TouchMirror: %v", m.Repo, err) @@ -420,6 +467,13 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { return false } + if ok := runSyncMisc(ctx, m); !ok { + if err = repo_model.TouchMirror(ctx, m); err != nil { + log.Error("SyncMirrors [repo: %-v]: failed to TouchMirror: %v", m.Repo, err) + } + return false + } + log.Trace("SyncMirrors [repo: %-v]: Scheduling next update", m.Repo) m.ScheduleNextUpdate() if err = repo_model.UpdateMirror(ctx, m); err != nil { From 78945ebd0efb9036094f5449bce98b1bf61015ab Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 6 May 2023 18:31:48 -0400 Subject: [PATCH 47/94] Enable github sync flag and fix panics --- services/migrations/github.go | 4 ++++ services/migrations/migrate.go | 10 +++++++++- services/mirror/mirror_pull.go | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/services/migrations/github.go b/services/migrations/github.go index 5f562384397ac..8b9a2f586348c 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -126,6 +126,10 @@ func (g *GithubDownloaderV3) String() string { return fmt.Sprintf("migration from github server %s %s/%s", g.baseURL, g.repoOwner, g.repoName) } +func (g *GithubDownloaderV3) SupportSyncing() bool { + return true +} + // ColorFormat provides a basic color format for a GithubDownloader func (g *GithubDownloaderV3) ColorFormat(s fmt.State) { if g == nil { diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 4c92e20fbb3d0..473026accc701 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -17,6 +17,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" system_model "code.gitea.io/gitea/models/system" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/hostmatcher" "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" @@ -484,7 +485,8 @@ func migrateRepository(doer *user_model.User, downloader base.Downloader, upload } // SyncRepository syncs a repository according MigrateOptions -func SyncRepository(ctx context.Context, doer *user_model.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger, lastSynced time.Time) (*repo_model.Repository, error) { +func SyncRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, opts base.MigrateOptions, messenger base.Messenger, lastSynced time.Time) (*repo_model.Repository, error) { + ownerName := repo.OwnerName downloader, err := newDownloader(ctx, ownerName, opts) if err != nil { return nil, err @@ -497,6 +499,12 @@ func SyncRepository(ctx context.Context, doer *user_model.User, ownerName string uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName) uploader.gitServiceType = opts.GitServiceType + uploader.repo = repo + uploader.gitRepo, err = git.OpenRepository(ctx, repo.RepoPath()) + if err != nil { + log.Error("open repository failed: %v", err) + return nil, err + } if err := syncRepository(downloader, uploader, opts, messenger, lastSynced); err != nil { if err1 := uploader.Rollback(); err1 != nil { diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 4d7cd03b2c745..7dc10909487bd 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -427,7 +427,7 @@ func runSyncMisc(ctx context.Context, m *repo_model.Mirror) bool { MigrateToRepoID: repo.ID, MirrorInterval: m.Interval.String(), } - _, err := migrations.SyncRepository(ctx, repo.Owner, repo.OwnerName, opts, nil, m.UpdatedUnix.AsTime()) + _, err := migrations.SyncRepository(ctx, repo.Owner, repo, opts, nil, m.UpdatedUnix.AsTime()) if err != nil { log.Error("SyncMirrors [repo: %-v]: failed to sync repository: %v", m.Repo, err) return false From a3a40157a3ffb2905307ea8e174dc94ff544cc31 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 6 May 2023 19:47:56 -0400 Subject: [PATCH 48/94] Correctly store mirror setting in migration --- services/migrations/gitea_uploader.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 19dcfae9354fd..a604172dec6b2 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -130,6 +130,12 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate Private: repo.IsPrivate, Wiki: opts.Wiki, Releases: opts.Releases, // if didn't get releases, then sync them from tags + Issues: opts.Issues, + Milestones: opts.Milestones, + Labels: opts.Labels, + Comments: opts.Comments, + PullRequests: opts.PullRequests, + ReleaseAssets: opts.ReleaseAssets, MirrorInterval: opts.MirrorInterval, }, NewMigrationHTTPTransport()) From b8f8ab43a63c2abab5628edcd26faf9c4c2e11fc Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 6 May 2023 21:24:06 -0400 Subject: [PATCH 49/94] Fix: don't delete repo when sync failed --- services/migrations/migrate.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 473026accc701..ea4eba946fb2e 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -507,11 +507,11 @@ func SyncRepository(ctx context.Context, doer *user_model.User, repo *repo_model } if err := syncRepository(downloader, uploader, opts, messenger, lastSynced); err != nil { - if err1 := uploader.Rollback(); err1 != nil { - log.Error("rollback failed: %v", err1) - } - if err2 := system_model.CreateRepositoryNotice(fmt.Sprintf("Syncing repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil { - log.Error("create repository notice failed: ", err2) + // It's different from migration that we shouldn't rollback here, + // because the only thing rollback does is to delete the repository + + if err := system_model.CreateRepositoryNotice(fmt.Sprintf("Syncing repository from %s failed: %v", opts.OriginalURL, err)); err != nil { + log.Error("create repository notice failed: ", err) } return nil, err } From de2f2a739ae042da55dfec9e92df1259f41f91d3 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 6 May 2023 22:06:47 -0400 Subject: [PATCH 50/94] Fix error when syncing milestones --- models/migrate.go | 79 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 17 deletions(-) diff --git a/models/migrate.go b/models/migrate.go index d61d80990818c..5e39d336ad547 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -53,32 +53,77 @@ func UpdateMilestones(ms ...*issues_model.Milestone) (err error) { defer committer.Close() sess := db.GetEngine(ctx) - // check if milestone exists, if not create it - // milestones are considered as unique by RepoID and CreatedUnix + // get existing milestones + existingMilestones := make([]*issues_model.Milestone, 0) + if err = sess.Where("repo_id = ?", ms[0].RepoID).Find(&existingMilestones); err != nil { + return err + } + + milestonesToAdd := make([]*issues_model.Milestone, 0) + milestonesToUpdate := make([]*issues_model.Milestone, 0) + milestonesToDelete := make([]*issues_model.Milestone, 0) + foundMap := make(map[int64]bool) + + openCount := 0 + closedCount := 0 + for _, m := range ms { - var existingMilestone *issues_model.Milestone - has, err := sess.Where("repo_id = ? AND created_unix = ?", m.RepoID, m.CreatedUnix).Get(&existingMilestone) - if err != nil { + var foundMilestone *issues_model.Milestone + for _, existingMilestone := range existingMilestones { + if existingMilestone.CreatedUnix == m.CreatedUnix { + foundMilestone = existingMilestone + foundMap[existingMilestone.ID] = true + break + } + } + + if foundMilestone == nil { + milestonesToAdd = append(milestonesToAdd, m) + } else if foundMilestone.UpdatedUnix != m.UpdatedUnix { + // consider as updated if updated_unix is different + m.ID = foundMilestone.ID + milestonesToUpdate = append(milestonesToUpdate, m) + } + + if m.IsClosed { + closedCount++ + } else { + openCount++ + } + } + + for _, existingMilestone := range existingMilestones { + if _, exist := foundMap[existingMilestone.ID]; !exist { + milestonesToDelete = append(milestonesToDelete, existingMilestone) + } + } + + if len(milestonesToAdd) > 0 { + if _, err = sess.Insert(milestonesToAdd); err != nil { return err } + } - if !has { - if _, err = sess.NoAutoTime().Insert(m); err != nil { - return err - } - } else if existingMilestone.Name != m.Name || existingMilestone.Content != m.Content || - existingMilestone.IsClosed != m.IsClosed || - existingMilestone.UpdatedUnix != m.UpdatedUnix || existingMilestone.ClosedDateUnix != m.ClosedDateUnix || - existingMilestone.DeadlineUnix != m.DeadlineUnix { - if _, err = sess.NoAutoTime().ID(existingMilestone.ID).Update(m); err != nil { - return err - } + for _, m := range milestonesToUpdate { + if _, err = sess.ID(m.ID).AllCols().Update(m); err != nil { + return err } } - if _, err = sess.Exec(ctx, "UPDATE `repository` SET num_milestones = num_milestones + ? WHERE id = ?", len(ms), ms[0].RepoID); err != nil { + for _, m := range milestonesToDelete { + if _, err = sess.ID(m.ID).Delete(m); err != nil { + return err + } + } + + if _, err = sess.ID(ms[0].RepoID).Update(&repo_model.Repository{ + NumMilestones: len(ms), + NumOpenMilestones: openCount, + NumClosedMilestones: closedCount, + }); err != nil { return err } + return committer.Commit() } From 3a72ed9835620f4af0fff78bee90df4f938d1525 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 6 May 2023 23:48:23 -0400 Subject: [PATCH 51/94] Update label identification method --- models/issues/label.go | 37 +++++++++++++++------------ modules/migration/label.go | 1 + services/migrations/gitea_uploader.go | 1 + services/migrations/github.go | 1 + 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/models/issues/label.go b/models/issues/label.go index 4335318d08f18..7f6e065970bd6 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -92,6 +92,8 @@ type Label struct { CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + OriginalID int64 // Only for migrating data from other system, used for syncing + NumOpenIssues int `xorm:"-"` NumOpenRepoIssues int64 `xorm:"-"` IsChecked bool `xorm:"-"` @@ -281,7 +283,9 @@ func deleteLabel(ctx context.Context, id, labelID int64) error { if _, err = sess.ID(labelID).Delete(new(Label)); err != nil { return err - } else if _, err = sess. + } + + if _, err = sess. Where("label_id = ?", labelID). Delete(new(IssueLabel)); err != nil { return err @@ -444,32 +448,31 @@ func UpdateLabelsByRepoID(repoID int64, labels ...*Label) error { labelsToDelete := make([]*Label, 0) for _, l := range labels { - color, err := label.NormalizeColor(l.Color) - if err != nil { - return err - } - l.Color = color - - found := false + var foundLabel *Label for _, existingLabel := range existingLabels { - if existingLabel.ID == l.ID { - found = true - if existingLabel.Name != l.Name || existingLabel.Description != l.Description || - existingLabel.Color != l.Color { - labelsToUpdate = append(labelsToUpdate, l) - } + if existingLabel.OriginalID == l.OriginalID { + foundLabel = existingLabel + break } } - if !found { + + if foundLabel == nil { labelsToAdd = append(labelsToAdd, l) + } else { + if foundLabel.Name != l.Name || foundLabel.Description != l.Description || + foundLabel.Color != l.Color { + l.RepoID = repoID + labelsToUpdate = append(labelsToUpdate, l) + } } } for _, existingLabel := range existingLabels { found := false for _, label := range labels { - if label.ID == existingLabel.ID { + if label.OriginalID == existingLabel.OriginalID { found = true + break } } if !found { @@ -495,7 +498,7 @@ func UpdateLabelsByRepoID(repoID int64, labels ...*Label) error { } } - return nil + return committer.Commit() } // CountLabelsByRepoID count number of all labels that belong to given repository by ID. diff --git a/modules/migration/label.go b/modules/migration/label.go index 4927be3c0bd11..1c857ea51efde 100644 --- a/modules/migration/label.go +++ b/modules/migration/label.go @@ -10,4 +10,5 @@ type Label struct { Color string `json:"color"` Description string `json:"description"` Exclusive bool `json:"exclusive"` + OriginalID int64 `json:"-"` // ID of the label from the original Git service } diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index a604172dec6b2..048c769e935c2 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -259,6 +259,7 @@ func (g *GiteaLocalUploader) convertLabels(labels ...*base.Label) []*issues_mode Exclusive: l.Exclusive, Description: l.Description, Color: l.Color, + OriginalID: l.OriginalID, }) } return lbs diff --git a/services/migrations/github.go b/services/migrations/github.go index 8b9a2f586348c..a57e2567a842d 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -282,6 +282,7 @@ func convertGithubLabel(label *github.Label) *base.Label { Name: label.GetName(), Color: label.GetColor(), Description: label.GetDescription(), + OriginalID: label.GetID(), } } From ef82e4e22cc59693bf9dcc645f69fb946721824a Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sun, 7 May 2023 18:20:29 -0400 Subject: [PATCH 52/94] Fix lint --- models/issues/label.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/models/issues/label.go b/models/issues/label.go index 7f6e065970bd6..0d7c069779a0d 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -458,12 +458,10 @@ func UpdateLabelsByRepoID(repoID int64, labels ...*Label) error { if foundLabel == nil { labelsToAdd = append(labelsToAdd, l) - } else { - if foundLabel.Name != l.Name || foundLabel.Description != l.Description || - foundLabel.Color != l.Color { - l.RepoID = repoID - labelsToUpdate = append(labelsToUpdate, l) - } + } else if foundLabel.Name != l.Name || foundLabel.Description != l.Description || + foundLabel.Color != l.Color { + l.RepoID = repoID + labelsToUpdate = append(labelsToUpdate, l) } } From e52964c2968bf169d13f1fc774b590200339a1a3 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Mon, 15 May 2023 11:23:40 +0800 Subject: [PATCH 53/94] Use correct last synced time --- services/mirror/mirror_pull.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 7dc10909487bd..71d10fe3cc975 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -391,7 +391,7 @@ func runSyncGit(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, // runSyncMisc runs the sync of Issues, Pull Requests, Reviews, Topics, Releases, Labels, and Milestones. // It returns true if the sync was successful. -func runSyncMisc(ctx context.Context, m *repo_model.Mirror) bool { +func runSyncMisc(ctx context.Context, m *repo_model.Mirror, lastSynced time.Time) bool { repo := m.GetRepository() remoteURL, remoteErr := git.GetRemoteURL(ctx, repo.RepoPath(), m.GetRemoteName()) @@ -427,7 +427,8 @@ func runSyncMisc(ctx context.Context, m *repo_model.Mirror) bool { MigrateToRepoID: repo.ID, MirrorInterval: m.Interval.String(), } - _, err := migrations.SyncRepository(ctx, repo.Owner, repo, opts, nil, m.UpdatedUnix.AsTime()) + + _, err := migrations.SyncRepository(ctx, m.Repo.MustOwner(ctx), repo, opts, nil, lastSynced) if err != nil { log.Error("SyncMirrors [repo: %-v]: failed to sync repository: %v", m.Repo, err) return false @@ -455,6 +456,9 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { } _ = m.GetRepository() // force load repository of mirror + // UpdatedUnix will be updated by runSyncGit, but we need to store it here to use it in runSyncMisc + lastSynced := m.UpdatedUnix.AsTime() + ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Syncing Mirror %s/%s", m.Repo.OwnerName, m.Repo.Name)) defer finished() @@ -467,7 +471,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { return false } - if ok := runSyncMisc(ctx, m); !ok { + if ok := runSyncMisc(ctx, m, lastSynced); !ok { if err = repo_model.TouchMirror(ctx, m); err != nil { log.Error("SyncMirrors [repo: %-v]: failed to TouchMirror: %v", m.Repo, err) } From 82fe9d5a84d385870164e388ef8a35cfce263029 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Mon, 15 May 2023 15:54:01 +0800 Subject: [PATCH 54/94] Correctly get issue id --- models/migrate.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/models/migrate.go b/models/migrate.go index 5e39d336ad547..93eda893ab8af 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -241,17 +241,17 @@ func updateIssue(ctx context.Context, issue *issues_model.Issue) error { func upsertIssue(ctx context.Context, issue *issues_model.Issue) (isInsert bool, err error) { sess := db.GetEngine(ctx) - exists, err := sess.Exist(&issues_model.Issue{ - RepoID: issue.RepoID, - Index: issue.Index, - }) + var issueIDs []int64 + err = sess.Table("issue").Where("repo_id = ? AND `index` = ?", issue.RepoID, issue.Index).Cols("id").Find(&issueIDs) if err != nil { return false, err } - if !exists { + if len(issueIDs) == 0 { return true, insertIssue(ctx, issue) } + + issue.ID = issueIDs[0] return false, updateIssue(ctx, issue) } From 81cf9fcebb9af9d675441ceddc5c7c4923ae2eff Mon Sep 17 00:00:00 2001 From: harryzcy Date: Tue, 16 May 2023 10:44:56 +0800 Subject: [PATCH 55/94] Fix PR request sync issue --- services/migrations/gitea_uploader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 048c769e935c2..aa3542b69ab8b 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -1059,7 +1059,7 @@ func (g *GiteaLocalUploader) PatchPullRequests(prs ...*base.PullRequest) error { if err != nil { return err } - if err := models.InsertPullRequests(gprs...); err != nil { + if err := models.UpsertPullRequests(gprs...); err != nil { return err } for _, pr := range gprs { From 6e3b98051bda428aced763c4d98e556205f4c0e6 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 24 Jun 2023 17:54:27 -0500 Subject: [PATCH 56/94] Add OriginalID for milestones --- models/issues/milestone.go | 2 ++ models/migrate.go | 6 +++--- modules/migration/milestone.go | 1 + services/migrations/gitea_uploader.go | 1 + services/migrations/github.go | 1 + 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/models/issues/milestone.go b/models/issues/milestone.go index ffe5c8eb509ba..12f59ac8d4e7d 100644 --- a/models/issues/milestone.go +++ b/models/issues/milestone.go @@ -64,6 +64,8 @@ type Milestone struct { DeadlineString string `xorm:"-"` TotalTrackedTime int64 `xorm:"-"` + + OriginalID int64 // ID from the upstream syncing source } func init() { diff --git a/models/migrate.go b/models/migrate.go index 93eda893ab8af..97be1367b4f5b 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -70,7 +70,7 @@ func UpdateMilestones(ms ...*issues_model.Milestone) (err error) { for _, m := range ms { var foundMilestone *issues_model.Milestone for _, existingMilestone := range existingMilestones { - if existingMilestone.CreatedUnix == m.CreatedUnix { + if existingMilestone.OriginalID == m.OriginalID { foundMilestone = existingMilestone foundMap[existingMilestone.ID] = true break @@ -79,8 +79,7 @@ func UpdateMilestones(ms ...*issues_model.Milestone) (err error) { if foundMilestone == nil { milestonesToAdd = append(milestonesToAdd, m) - } else if foundMilestone.UpdatedUnix != m.UpdatedUnix { - // consider as updated if updated_unix is different + } else if foundMilestone.OriginalID != m.OriginalID { m.ID = foundMilestone.ID milestonesToUpdate = append(milestonesToUpdate, m) } @@ -116,6 +115,7 @@ func UpdateMilestones(ms ...*issues_model.Milestone) (err error) { } } + // TODO: is this correct? if _, err = sess.ID(ms[0].RepoID).Update(&repo_model.Repository{ NumMilestones: len(ms), NumOpenMilestones: openCount, diff --git a/modules/migration/milestone.go b/modules/migration/milestone.go index 34355b8f9c8aa..41b8fd7dfe379 100644 --- a/modules/migration/milestone.go +++ b/modules/migration/milestone.go @@ -15,4 +15,5 @@ type Milestone struct { Updated *time.Time `json:"updated"` Closed *time.Time `json:"closed"` State string `json:"state"` // open, closed + OriginalID int64 `json:"-"` // ID from the upstream syncing source } diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 8e32f5eb7c34f..ffb567efc0fe4 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -208,6 +208,7 @@ func (g *GiteaLocalUploader) prepareMilestones(milestones ...*base.Milestone) [] CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()), UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()), DeadlineUnix: deadline, + OriginalID: milestone.OriginalID, } if ms.IsClosed && milestone.Closed != nil { ms.ClosedDateUnix = timeutil.TimeStamp(milestone.Closed.Unix()) diff --git a/services/migrations/github.go b/services/migrations/github.go index db7f704072e41..b72ee5f870adc 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -265,6 +265,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { Created: m.GetCreatedAt().Time, Updated: m.UpdatedAt.GetTime(), Closed: m.ClosedAt.GetTime(), + OriginalID: m.GetID(), }) } if len(ms) < perPage { From 186a9a932f48176d3984fa21a4c767142013adc4 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 24 Jun 2023 18:03:13 -0500 Subject: [PATCH 57/94] Add OriginalID to comments --- models/issues/comment.go | 1 + models/migrate.go | 2 +- modules/migration/comment.go | 1 + services/migrations/gitea_uploader.go | 1 + services/migrations/github.go | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/models/issues/comment.go b/models/issues/comment.go index dbe4434ca7542..d436e96e36bc6 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -225,6 +225,7 @@ type Comment struct { Poster *user_model.User `xorm:"-"` OriginalAuthor string OriginalAuthorID int64 + OriginalID int64 // Only used in synced comments IssueID int64 `xorm:"INDEX"` Issue *Issue `xorm:"-"` LabelID int64 diff --git a/models/migrate.go b/models/migrate.go index 97be1367b4f5b..6c5d845072efb 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -328,7 +328,7 @@ func UpsertIssueComments(comments []*issues_model.Comment) error { } } else { if _, err := sess.NoAutoTime().Where( - "issue_id = ? AND created_unix = ?", comment.IssueID, comment.CreatedUnix, + "issue_id = ? AND original_id = ?", comment.IssueID, comment.OriginalID, ).Update(comment); err != nil { return err } diff --git a/modules/migration/comment.go b/modules/migration/comment.go index 92ce30e302023..bd1a8a9b66462 100644 --- a/modules/migration/comment.go +++ b/modules/migration/comment.go @@ -25,6 +25,7 @@ type Comment struct { Content string Reactions []*Reaction Meta map[string]interface{} `yaml:"meta,omitempty"` // see models/issues/comment.go for fields in Comment struct + OriginalID int64 `yaml:"-"` // ID from the upstream syncing source } // GetExternalName ExternalUserMigrated interface diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index ffb567efc0fe4..ddd86d0f4a84b 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -506,6 +506,7 @@ func (g *GiteaLocalUploader) prepareComments(comments ...*base.Comment) ([]*issu Content: comment.Content, CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()), UpdatedUnix: timeutil.TimeStamp(comment.Updated.Unix()), + OriginalID: comment.OriginalID, } switch cm.Type { diff --git a/services/migrations/github.go b/services/migrations/github.go index b72ee5f870adc..0f482ce8ff7f9 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -490,6 +490,7 @@ func (g *GithubDownloaderV3) getCommentsSince(commentable base.Commentable, sinc Created: comment.GetCreatedAt().Time, Updated: comment.GetUpdatedAt().Time, Reactions: reactions, + OriginalID: comment.GetID(), }) } if resp.NextPage == 0 { From 837c09e65e0f339188b3ad5daf2646f735ccb5cf Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 24 Jun 2023 18:03:51 -0500 Subject: [PATCH 58/94] Update comment for label struct --- modules/migration/label.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migration/label.go b/modules/migration/label.go index 1c857ea51efde..e061f1a4d6894 100644 --- a/modules/migration/label.go +++ b/modules/migration/label.go @@ -10,5 +10,5 @@ type Label struct { Color string `json:"color"` Description string `json:"description"` Exclusive bool `json:"exclusive"` - OriginalID int64 `json:"-"` // ID of the label from the original Git service + OriginalID int64 `json:"-"` // ID from the upstream syncing source } From cbed118a30f6dcf9996c151d61c1595186449b07 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 21 Jul 2023 01:58:36 -0500 Subject: [PATCH 59/94] Add OriginalID to Release --- modules/migration/release.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/migration/release.go b/modules/migration/release.go index f92cf25e7bc5d..ef221b992113b 100644 --- a/modules/migration/release.go +++ b/modules/migration/release.go @@ -37,6 +37,7 @@ type Release struct { Assets []*ReleaseAsset Created time.Time Published time.Time + OriginalID int64 `yaml:"-"` // ID from the upstream syncing source } // GetExternalName ExternalUserMigrated interface From 2b31c3354f123dd14c775c8bd6982fa5bbe054eb Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 21 Jul 2023 02:03:47 -0500 Subject: [PATCH 60/94] Won't hit rate limit with github actions --- services/migrations/gitea_uploader_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index 878b6d6b8453f..86676b7050ec0 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -30,9 +30,6 @@ import ( ) func TestGiteaUploadRepo(t *testing.T) { - // FIXME: Since no accesskey or user/password will trigger rate limit of github, just skip - t.Skip() - unittest.PrepareTestEnv(t) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) From 3d9b7edd35319a0f2c8822205e7f8467aa144017 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 21 Jul 2023 19:06:18 -0500 Subject: [PATCH 61/94] Fix: include originalID in getAllCommentsSince --- services/migrations/github.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/migrations/github.go b/services/migrations/github.go index 0f482ce8ff7f9..92edc7ef48d36 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -574,6 +574,7 @@ func (g *GithubDownloaderV3) getAllCommentsSince(page, perPage int, since *time. Created: comment.GetCreatedAt().Time, Updated: comment.GetUpdatedAt().Time, Reactions: reactions, + OriginalID: comment.GetID(), }) } From f7e94224d6d29f4ef90f4b4d75d9eb3ce0071173 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 21 Jul 2023 19:32:22 -0500 Subject: [PATCH 62/94] Identify comment by original_id only --- models/migrate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrate.go b/models/migrate.go index 6c5d845072efb..f8cbf8ec3a8d8 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -328,7 +328,7 @@ func UpsertIssueComments(comments []*issues_model.Comment) error { } } else { if _, err := sess.NoAutoTime().Where( - "issue_id = ? AND original_id = ?", comment.IssueID, comment.OriginalID, + "original_id = ?", comment.IssueID, comment.OriginalID, ).Update(comment); err != nil { return err } From 328fdafdca93c3d54ba452f3c0a2cb9c8905f305 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 21 Jul 2023 19:57:02 -0500 Subject: [PATCH 63/94] Add OriginalID to reviews and review comments --- models/issues/review.go | 14 ++++++++------ models/migrate.go | 2 +- modules/migration/review.go | 26 ++++++++++++++------------ services/migrations/gitea_uploader.go | 3 ++- services/migrations/github.go | 24 +++++++++++++----------- 5 files changed, 38 insertions(+), 31 deletions(-) diff --git a/models/issues/review.go b/models/issues/review.go index e107ef354bcee..8dd6bc8c54d54 100644 --- a/models/issues/review.go +++ b/models/issues/review.go @@ -111,6 +111,7 @@ type Review struct { ReviewerTeam *organization.Team `xorm:"-"` OriginalAuthor string OriginalAuthorID int64 + OriginalID int64 Issue *Issue `xorm:"-"` IssueID int64 `xorm:"index"` Content string `xorm:"TEXT"` @@ -557,7 +558,7 @@ func UpsertReviews(reviews []*Review) error { sess := db.GetEngine(ctx) for _, review := range reviews { - exists, err := sess.Where("issue_id = ? AND created_unix", review.IssueID, review.CreatedUnix).Exist(&Review{}) + exists, err := sess.Where("original_id = ?", review.OriginalID).Exist(&Review{}) if err != nil { return err } @@ -581,12 +582,12 @@ func UpsertReviews(reviews []*Review) error { } } } else { - if _, err = sess.NoAutoTime().Where("issue_id = ? AND created_unix = ?", review.IssueID, review.CreatedUnix).Update(review); err != nil { + if _, err = sess.NoAutoTime().Where("original_id = ?", review.OriginalID).Update(review); err != nil { return err } // Get id of the review - if err = sess.NoAutoTime().Where("issue_id = ? AND created_unix = ?", review.IssueID, review.CreatedUnix).Find(review); err != nil { + if err = sess.NoAutoTime().Where("original_id = ?", review.OriginalID).Find(review); err != nil { return err } @@ -601,7 +602,7 @@ func UpsertReviews(reviews []*Review) error { return err } } else { - if _, err := sess.NoAutoTime().Where("review_id = ? AND created_unix = ?", review.ID, comment.CreatedUnix).Update(comment); err != nil { + if _, err := sess.NoAutoTime().Where("original_id = ?", comment.OriginalID).Update(comment); err != nil { return err } } @@ -622,7 +623,7 @@ func UpsertReviews(reviews []*Review) error { return err } } else { - if _, err := sess.NoAutoTime().Where("review_id = ? AND created_unix = ?", review.ID, comment.CreatedUnix).Update(comment); err != nil { + if _, err := sess.NoAutoTime().Where("original_id = ?", comment.OriginalID).Update(comment); err != nil { return err } } @@ -635,7 +636,7 @@ func UpsertReviews(reviews []*Review) error { } func existsCommentByReviewIDAndCreatedUnix(sess db.Engine, comment *Comment) (bool, error) { - return sess.Where("review_id = ? AND created_unix = ?", comment.ReviewID, comment.CreatedUnix).Exist(&Comment{}) + return sess.Where("original_id = ?", comment.OriginalID).Exist(&Comment{}) } func generateCommentFromReview(review *Review) *Comment { @@ -649,6 +650,7 @@ func generateCommentFromReview(review *Review) *Comment { ReviewID: review.ID, CreatedUnix: review.CreatedUnix, UpdatedUnix: review.UpdatedUnix, + OriginalID: review.OriginalID, } } diff --git a/models/migrate.go b/models/migrate.go index f8cbf8ec3a8d8..cd59797df31d2 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -366,7 +366,7 @@ func UpsertIssueComments(comments []*issues_model.Comment) error { } for issueID := range issueIDs { - if _, err := db.Exec(ctx, "UPDATE issue set num_comments = (SELECT count(*) FROM comment WHERE issue_id = ? AND `type`=?) WHERE id = ?", + if _, err := db.Exec(ctx, "UPDATE issue SET num_comments = (SELECT count(*) FROM comment WHERE issue_id = ? AND `type`=?) WHERE id = ?", issueID, issues_model.CommentTypeComment, issueID); err != nil { return err } diff --git a/modules/migration/review.go b/modules/migration/review.go index 79e821b2e108a..a778f84e54f9b 100644 --- a/modules/migration/review.go +++ b/modules/migration/review.go @@ -42,6 +42,7 @@ type Review struct { CreatedAt time.Time `yaml:"created_at"` State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT Comments []*ReviewComment + OriginalID int64 } // GetExternalName ExternalUserMigrated interface @@ -52,16 +53,17 @@ func (r *Review) GetExternalID() int64 { return r.ReviewerID } // ReviewComment represents a review comment type ReviewComment struct { - ID int64 - InReplyTo int64 `yaml:"in_reply_to"` - Content string - TreePath string `yaml:"tree_path"` - DiffHunk string `yaml:"diff_hunk"` - Position int - Line int - CommitID string `yaml:"commit_id"` - PosterID int64 `yaml:"poster_id"` - Reactions []*Reaction - CreatedAt time.Time `yaml:"created_at"` - UpdatedAt time.Time `yaml:"updated_at"` + ID int64 + InReplyTo int64 `yaml:"in_reply_to"` + Content string + TreePath string `yaml:"tree_path"` + DiffHunk string `yaml:"diff_hunk"` + Position int + Line int + CommitID string `yaml:"commit_id"` + PosterID int64 `yaml:"poster_id"` + Reactions []*Reaction + CreatedAt time.Time `yaml:"created_at"` + UpdatedAt time.Time `yaml:"updated_at"` + OriginalID int64 } diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index ddd86d0f4a84b..af591345f5768 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -883,6 +883,7 @@ func (g *GiteaLocalUploader) prepareReviews(reviews ...*base.Review) ([]*issues_ Official: review.Official, CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()), UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()), + OriginalID: review.OriginalID, } if err := g.remapUser(review, &cm); err != nil { @@ -902,7 +903,7 @@ func (g *GiteaLocalUploader) prepareReviews(reviews ...*base.Review) ([]*issues_ g.prCache[issue.ID] = pr } if pr.MergeBase == "" { - // No mergebase -> no basis for any patches + // No merge base -> no basis for any patches log.Warn("PR #%d in %s/%s: does not have a merge base, all review comments will be ignored", pr.Index, g.repoOwner, g.repoName) continue } diff --git a/services/migrations/github.go b/services/migrations/github.go index 92edc7ef48d36..ddeba0c911e6f 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -596,6 +596,7 @@ func (g *GithubDownloaderV3) convertGithubReview(r *github.PullRequestReview) *b Content: r.GetBody(), CreatedAt: r.GetSubmittedAt().Time, State: r.GetState(), + OriginalID: r.GetID(), } } @@ -629,17 +630,18 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques } rcs = append(rcs, &base.ReviewComment{ - ID: c.GetID(), - InReplyTo: c.GetInReplyTo(), - Content: c.GetBody(), - TreePath: c.GetPath(), - DiffHunk: c.GetDiffHunk(), - Position: c.GetPosition(), - CommitID: c.GetCommitID(), - PosterID: c.GetUser().GetID(), - Reactions: reactions, - CreatedAt: c.GetCreatedAt().Time, - UpdatedAt: c.GetUpdatedAt().Time, + ID: c.GetID(), + InReplyTo: c.GetInReplyTo(), + Content: c.GetBody(), + TreePath: c.GetPath(), + DiffHunk: c.GetDiffHunk(), + Position: c.GetPosition(), + CommitID: c.GetCommitID(), + PosterID: c.GetUser().GetID(), + Reactions: reactions, + CreatedAt: c.GetCreatedAt().Time, + UpdatedAt: c.GetUpdatedAt().Time, + OriginalID: c.GetID(), }) } return rcs, nil From 87c92384dfff8fe50414d1ce50ad8dfbfc5a69ec Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 21 Jul 2023 21:29:16 -0500 Subject: [PATCH 64/94] Enable TestGiteaUploadRepo only when token --- services/migrations/gitea_uploader_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index 86676b7050ec0..47068f724bc71 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -30,13 +30,18 @@ import ( ) func TestGiteaUploadRepo(t *testing.T) { + token := os.Getenv("GITHUB_READ_TOKEN") + if token == "" { + t.Skip("Skipping GitHub migration test because GITHUB_READ_TOKEN is empty") + } + unittest.PrepareTestEnv(t) user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) var ( ctx = context.Background() - downloader = NewGithubDownloaderV3(ctx, "https://github.com", "", "", "", "go-xorm", "builder") + downloader = NewGithubDownloaderV3(ctx, "https://github.com", "", "", token, "go-xorm", "builder") repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05") uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName) ) From a48b1f391913c2e0892f1dc3fece06015ade4055 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 21 Jul 2023 22:48:04 -0500 Subject: [PATCH 65/94] Fix: update all cols including false boolean value --- models/migrate.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/models/migrate.go b/models/migrate.go index cd59797df31d2..bc46e0e1a425b 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -115,7 +115,6 @@ func UpdateMilestones(ms ...*issues_model.Milestone) (err error) { } } - // TODO: is this correct? if _, err = sess.ID(ms[0].RepoID).Update(&repo_model.Repository{ NumMilestones: len(ms), NumOpenMilestones: openCount, @@ -197,7 +196,7 @@ func UpsertIssues(issues ...*issues_model.Issue) error { func updateIssue(ctx context.Context, issue *issues_model.Issue) error { sess := db.GetEngine(ctx) - if _, err := sess.NoAutoTime().ID(issue.ID).Update(issue); err != nil { + if _, err := sess.NoAutoTime().ID(issue.ID).AllCols().Update(issue); err != nil { return err } issueLabels := resolveIssueLabels(issue.ID, issue.Labels) @@ -224,7 +223,7 @@ func updateIssue(ctx context.Context, issue *issues_model.Issue) error { return err } if exists { - if _, err := sess.ID(reaction.ID).Update(&reaction); err != nil { + if _, err := sess.ID(reaction.ID).AllCols().Update(&reaction); err != nil { return err } } else { @@ -329,7 +328,7 @@ func UpsertIssueComments(comments []*issues_model.Comment) error { } else { if _, err := sess.NoAutoTime().Where( "original_id = ?", comment.IssueID, comment.OriginalID, - ).Update(comment); err != nil { + ).AllCols().Update(comment); err != nil { return err } } @@ -353,7 +352,7 @@ func UpsertIssueComments(comments []*issues_model.Comment) error { if _, err := sess.Where( "issue_id = ? AND comment_id = ? AND type = ?", reaction.IssueID, reaction.CommentID, reaction.Type, - ).Update(&reaction); err != nil { + ).AllCols().Update(&reaction); err != nil { return err } } else { @@ -414,7 +413,7 @@ func UpsertPullRequests(prs ...*issues_model.PullRequest) error { return err } } else { - if _, err := sess.NoAutoTime().ID(pr.ID).Update(pr); err != nil { + if _, err := sess.NoAutoTime().ID(pr.ID).AllCols().Update(pr); err != nil { return err } } @@ -480,7 +479,9 @@ func UpsertReleases(rels ...*repo_model.Release) error { } } } else { - if _, err := sess.NoAutoTime().Where("repo_id = ? AND tag_name = ?", rel.RepoID, rel.TagName).Update(rel); err != nil { + if _, err := sess.NoAutoTime(). + Where("repo_id = ? AND tag_name = ?", rel.RepoID, rel.TagName). + AllCols().Update(rel); err != nil { return err } From 0fedce2beb5677dec1461c8bd35765bd9d274086 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 22 Jul 2023 01:27:13 -0500 Subject: [PATCH 66/94] Comment out code that cause bug with releases --- services/migrations/migrate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index ea4eba946fb2e..ed617643c02cb 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -611,9 +611,9 @@ func syncRepository(downloader base.Downloader, uploader base.Uploader, opts bas } // Once all releases (if any) are inserted, sync any remaining non-release tags - if err = uploader.SyncTags(); err != nil { - return err - } + // if err = uploader.SyncTags(); err != nil { + // return err + // } } var ( From d8c06346f10859e497f076cb63daaac9cf13e2f9 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Sat, 22 Jul 2023 01:28:15 -0500 Subject: [PATCH 67/94] Another place to comment out --- services/migrations/migrate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index ed617643c02cb..cc09b5a9ed71d 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -314,9 +314,9 @@ func migrateRepository(doer *user_model.User, downloader base.Downloader, upload } // Once all releases (if any) are inserted, sync any remaining non-release tags - if err = uploader.SyncTags(); err != nil { - return err - } + // if err = uploader.SyncTags(); err != nil { + // return err + // } } var ( From bcb8866b67408c3e1557b3f0638d2c2501b6b261 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 22 Aug 2023 14:14:56 -0400 Subject: [PATCH 68/94] Temp disable fast tag syncing for mirrors --- modules/repository/repo.go | 6 +++--- services/migrations/migrate.go | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index a927f45c06570..99a33f66ec69e 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -292,9 +292,9 @@ func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) // optimized procedure for pull-mirrors which saves a lot of time (in // particular for repos with many tags). - if repo.IsMirror { - return pullMirrorReleaseSync(repo, gitRepo) - } + // if repo.IsMirror { + // return pullMirrorReleaseSync(repo, gitRepo) + // } existingRelTags := make(container.Set[string]) opts := repo_model.FindReleasesOptions{ diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index cc09b5a9ed71d..ea4eba946fb2e 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -314,9 +314,9 @@ func migrateRepository(doer *user_model.User, downloader base.Downloader, upload } // Once all releases (if any) are inserted, sync any remaining non-release tags - // if err = uploader.SyncTags(); err != nil { - // return err - // } + if err = uploader.SyncTags(); err != nil { + return err + } } var ( @@ -611,9 +611,9 @@ func syncRepository(downloader base.Downloader, uploader base.Uploader, opts bas } // Once all releases (if any) are inserted, sync any remaining non-release tags - // if err = uploader.SyncTags(); err != nil { - // return err - // } + if err = uploader.SyncTags(); err != nil { + return err + } } var ( From 80a9877d13bd5c5d9d9b26f25ecb39aa244e5390 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 22 Aug 2023 17:59:38 -0400 Subject: [PATCH 69/94] Refactor release syncing --- modules/repository/repo.go | 27 +++++++++++++++++---------- services/migrations/gitea_uploader.go | 4 ++-- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 99a33f66ec69e..be94245a488b3 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -286,15 +286,22 @@ func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo return repo, UpdateRepository(ctx, repo, false) } -// SyncReleasesWithTags synchronizes release table with repository tags -func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) error { - log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) +// SyncReleases synchronizes release table with repository tags +func SyncReleases(repo *repo_model.Repository, gitRepo *git.Repository, hasNonTagReleases bool) error { + log.Debug("SyncReleases: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) // optimized procedure for pull-mirrors which saves a lot of time (in // particular for repos with many tags). - // if repo.IsMirror { - // return pullMirrorReleaseSync(repo, gitRepo) - // } + if repo.IsMirror && !hasNonTagReleases { + return recreateMirrorReleaseFromTags(repo, gitRepo) + } + + return SyncReleasesWithTags(repo, gitRepo) +} + +// SyncReleasesWithTags synchronizes release table with repository tags for each of the releases +func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) error { + log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) existingRelTags := make(container.Set[string]) opts := repo_model.FindReleasesOptions{ @@ -493,14 +500,14 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re return nil } -// pullMirrorReleaseSync is a pull-mirror specific tag<->release table +// recreateMirrorReleaseFromTags is a pull-mirror specific tag<->release table // synchronization which overwrites all Releases from the repository tags. This // can be relied on since a pull-mirror is always identical to its // upstream. Hence, after each sync we want the pull-mirror release set to be // identical to the upstream tag set. This is much more efficient for // repositories like https://github.com/vim/vim (with over 13000 tags). -func pullMirrorReleaseSync(repo *repo_model.Repository, gitRepo *git.Repository) error { - log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) +func recreateMirrorReleaseFromTags(repo *repo_model.Repository, gitRepo *git.Repository) error { + log.Trace("recreateMirrorReleaseFromTags: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) tags, numTags, err := gitRepo.GetTagInfos(0, 0) if err != nil { return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) @@ -538,6 +545,6 @@ func pullMirrorReleaseSync(repo *repo_model.Repository, gitRepo *git.Repository) return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) } - log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags) + log.Trace("recreateMirrorReleaseFromTags: done rebuilding %d releases", numTags) return nil } diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 9200056a54d72..7b7b0968ed04b 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -381,9 +381,9 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { return models.InsertReleases(rels...) } -// SyncTags syncs releases with tags in the database +// SyncTags syncs releases with tags in the databases func (g *GiteaLocalUploader) SyncTags() error { - return repo_module.SyncReleasesWithTags(g.repo, g.gitRepo) + return repo_module.SyncReleases(g.repo, g.gitRepo, false) } func (g *GiteaLocalUploader) prepareIssues(issues ...*base.Issue) ([]*issues_model.Issue, error) { From c535198e6c47f921f850de80c0dc8f271ce182cb Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 22 Aug 2023 21:02:29 -0400 Subject: [PATCH 70/94] Refactor again --- cmd/admin.go | 2 +- modules/repository/repo.go | 15 ++++----------- services/migrations/gitea_uploader.go | 2 +- services/mirror/mirror_pull.go | 2 +- services/repository/fork.go | 2 +- 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/cmd/admin.go b/cmd/admin.go index a4f48b0513008..e7857f5da5a12 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -389,7 +389,7 @@ func runRepoSyncReleases(_ *cli.Context) error { } log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum) - if err = repo_module.SyncReleasesWithTags(repo, gitRepo); err != nil { + if err = repo_module.SyncReleasesWithTags(repo, gitRepo, true); err != nil { log.Warn(" SyncReleasesWithTags: %v", err) gitRepo.Close() continue diff --git a/modules/repository/repo.go b/modules/repository/repo.go index be94245a488b3..f6056ca75fb5f 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -159,7 +159,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, // note: this will greatly improve release (tag) sync // for pull-mirrors with many tags repo.IsMirror = opts.Mirror - if err = SyncReleasesWithTags(repo, gitRepo); err != nil { + if err = SyncReleasesWithTags(repo, gitRepo, false); err != nil { log.Error("Failed to synchronize tags to releases for repository: %v", err) } } @@ -286,9 +286,9 @@ func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo return repo, UpdateRepository(ctx, repo, false) } -// SyncReleases synchronizes release table with repository tags -func SyncReleases(repo *repo_model.Repository, gitRepo *git.Repository, hasNonTagReleases bool) error { - log.Debug("SyncReleases: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) +// SyncReleasesWithTags synchronizes release table with repository tags for each of the releases +func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository, hasNonTagReleases bool) error { + log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) // optimized procedure for pull-mirrors which saves a lot of time (in // particular for repos with many tags). @@ -296,13 +296,6 @@ func SyncReleases(repo *repo_model.Repository, gitRepo *git.Repository, hasNonTa return recreateMirrorReleaseFromTags(repo, gitRepo) } - return SyncReleasesWithTags(repo, gitRepo) -} - -// SyncReleasesWithTags synchronizes release table with repository tags for each of the releases -func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) error { - log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) - existingRelTags := make(container.Set[string]) opts := repo_model.FindReleasesOptions{ IncludeDrafts: true, diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 7b7b0968ed04b..92efa3be362f8 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -383,7 +383,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { // SyncTags syncs releases with tags in the databases func (g *GiteaLocalUploader) SyncTags() error { - return repo_module.SyncReleases(g.repo, g.gitRepo, false) + return repo_module.SyncReleasesWithTags(g.repo, g.gitRepo, false) } func (g *GiteaLocalUploader) prepareIssues(issues ...*base.Issue) ([]*issues_model.Issue, error) { diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 3c33574c8799b..b68238bc934b4 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -315,7 +315,7 @@ func runSyncGit(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, } log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) - if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { + if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo, false); err != nil { log.Error("SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v", m.Repo, err) } diff --git a/services/repository/fork.go b/services/repository/fork.go index 59aa173373314..84989d19a4f5e 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -186,7 +186,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork log.Error("Open created git repository failed: %v", err) } else { defer gitRepo.Close() - if err := repo_module.SyncReleasesWithTags(repo, gitRepo); err != nil { + if err := repo_module.SyncReleasesWithTags(repo, gitRepo, false); err != nil { log.Error("Sync releases from git tags failed: %v", err) } } From 6c99fbb9085a85948c049a6b7882365305f80123 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 22 Aug 2023 21:18:43 -0400 Subject: [PATCH 71/94] Fix bug --- services/migrations/gitea_uploader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 92efa3be362f8..28dd0e74b6fa4 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -383,7 +383,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { // SyncTags syncs releases with tags in the databases func (g *GiteaLocalUploader) SyncTags() error { - return repo_module.SyncReleasesWithTags(g.repo, g.gitRepo, false) + return repo_module.SyncReleasesWithTags(g.repo, g.gitRepo, true) } func (g *GiteaLocalUploader) prepareIssues(issues ...*base.Issue) ([]*issues_model.Issue, error) { From e199b95450751be533264ea3372fa17e669ec822 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 22 Aug 2023 21:30:51 -0400 Subject: [PATCH 72/94] Flip bool and add comments --- cmd/admin.go | 3 ++- modules/repository/repo.go | 11 +++++++---- services/migrations/gitea_uploader.go | 3 ++- services/mirror/mirror_pull.go | 3 ++- services/repository/fork.go | 3 ++- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/cmd/admin.go b/cmd/admin.go index e7857f5da5a12..5aa7abd3388af 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -389,7 +389,8 @@ func runRepoSyncReleases(_ *cli.Context) error { } log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum) - if err = repo_module.SyncReleasesWithTags(repo, gitRepo, true); err != nil { + tagOnlyReleases := false + if err = repo_module.SyncReleasesWithTags(repo, gitRepo, tagOnlyReleases); err != nil { log.Warn(" SyncReleasesWithTags: %v", err) gitRepo.Close() continue diff --git a/modules/repository/repo.go b/modules/repository/repo.go index f6056ca75fb5f..0e86044ea580a 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -159,7 +159,8 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, // note: this will greatly improve release (tag) sync // for pull-mirrors with many tags repo.IsMirror = opts.Mirror - if err = SyncReleasesWithTags(repo, gitRepo, false); err != nil { + tagOnlyReleases := true + if err = SyncReleasesWithTags(repo, gitRepo, tagOnlyReleases); err != nil { log.Error("Failed to synchronize tags to releases for repository: %v", err) } } @@ -286,13 +287,15 @@ func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo return repo, UpdateRepository(ctx, repo, false) } -// SyncReleasesWithTags synchronizes release table with repository tags for each of the releases -func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository, hasNonTagReleases bool) error { +// SyncReleasesWithTags synchronizes release table with repository tags for each of the releases. +// +// If tagOnlyReleases is true, then it is assumed all releases come from tags. +func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository, tagOnlyReleases bool) error { log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) // optimized procedure for pull-mirrors which saves a lot of time (in // particular for repos with many tags). - if repo.IsMirror && !hasNonTagReleases { + if repo.IsMirror && tagOnlyReleases { return recreateMirrorReleaseFromTags(repo, gitRepo) } diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 28dd0e74b6fa4..ccd84098506b1 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -383,7 +383,8 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { // SyncTags syncs releases with tags in the databases func (g *GiteaLocalUploader) SyncTags() error { - return repo_module.SyncReleasesWithTags(g.repo, g.gitRepo, true) + tagOnlyReleases := false + return repo_module.SyncReleasesWithTags(g.repo, g.gitRepo, tagOnlyReleases) } func (g *GiteaLocalUploader) prepareIssues(issues ...*base.Issue) ([]*issues_model.Issue, error) { diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index b68238bc934b4..1f6292d758766 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -315,7 +315,8 @@ func runSyncGit(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, } log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) - if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo, false); err != nil { + tagOnlyReleases := false + if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo, tagOnlyReleases); err != nil { log.Error("SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v", m.Repo, err) } diff --git a/services/repository/fork.go b/services/repository/fork.go index 84989d19a4f5e..a45ca922f1ac8 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -186,7 +186,8 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork log.Error("Open created git repository failed: %v", err) } else { defer gitRepo.Close() - if err := repo_module.SyncReleasesWithTags(repo, gitRepo, false); err != nil { + tagOnlyReleases := false + if err := repo_module.SyncReleasesWithTags(repo, gitRepo, tagOnlyReleases); err != nil { log.Error("Sync releases from git tags failed: %v", err) } } From 5169422a3d171f160ca47d5afbf8ae4e13fb9fea Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 22 Aug 2023 22:03:23 -0400 Subject: [PATCH 73/94] Make some error messages lowercased --- modules/repository/repo.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 0e86044ea580a..93ccdaba3335b 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -67,7 +67,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, var err error if err = util.RemoveAll(repoPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", repoPath, err) + return repo, fmt.Errorf("failed to remove %s: %w", repoPath, err) } if err = git.Clone(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{ @@ -77,9 +77,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, SkipTLSVerify: setting.Migrations.SkipTLSVerify, }); err != nil { if errors.Is(err, context.DeadlineExceeded) { - return repo, fmt.Errorf("Clone timed out. Consider increasing [git.timeout] MIGRATE in app.ini. Underlying Error: %w", err) + return repo, fmt.Errorf("clone timed out. Consider increasing [git.timeout] MIGRATE in app.ini. Underlying Error: %w", err) } - return repo, fmt.Errorf("Clone: %w", err) + return repo, fmt.Errorf("clone: %w", err) } if err := git.WriteCommitGraph(ctx, repoPath); err != nil { @@ -91,7 +91,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, wikiRemotePath := WikiRemoteURL(ctx, opts.CloneAddr) if len(wikiRemotePath) > 0 { if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + return repo, fmt.Errorf("failed to remove %s: %w", wikiPath, err) } if err := git.Clone(ctx, wikiRemotePath, wikiPath, git.CloneRepoOptions{ @@ -103,7 +103,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, }); err != nil { log.Warn("Clone wiki: %v", err) if err := util.RemoveAll(wikiPath); err != nil { - return repo, fmt.Errorf("Failed to remove %s: %w", wikiPath, err) + return repo, fmt.Errorf("failed to remove %s: %w", wikiPath, err) } } else { if err := git.WriteCommitGraph(ctx, wikiPath); err != nil { From 36c2378ace83f46d8ff0a2ac83b9e20e787c0b89 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 24 Aug 2023 01:02:22 -0400 Subject: [PATCH 74/94] Expose context --- models/issues/label.go | 78 +++++++++++++-------------------- models/issues/label_test.go | 8 ++-- routers/api/v1/org/label.go | 4 +- routers/api/v1/repo/label.go | 4 +- routers/web/org/org_labels.go | 4 +- routers/web/repo/issue_label.go | 4 +- 6 files changed, 43 insertions(+), 59 deletions(-) diff --git a/models/issues/label.go b/models/issues/label.go index 81ad7b947527c..d7a116337e379 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -222,11 +222,7 @@ func NewLabels(labels ...*Label) error { } // UpdateLabel updates label information. -func UpdateLabel(l *Label) error { - return updateLabel(db.DefaultContext, l) -} - -func updateLabel(ctx context.Context, l *Label) error { +func UpdateLabel(ctx context.Context, l *Label) error { color, err := label.NormalizeColor(l.Color) if err != nil { return err @@ -237,54 +233,42 @@ func updateLabel(ctx context.Context, l *Label) error { } // DeleteLabel delete a label -func DeleteLabel(id, labelID int64) error { - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - - if err = deleteLabel(ctx, id, labelID); err != nil { - return err - } +func DeleteLabel(ctx context.Context, id, labelID int64) error { + return db.WithTx(ctx, func(ctx context.Context) error { + l, err := GetLabelByID(ctx, labelID) + if err != nil { + if IsErrLabelNotExist(err) { + return nil + } + return err + } - return committer.Commit() -} + sess := db.GetEngine(ctx) -func deleteLabel(ctx context.Context, id, labelID int64) error { - l, err := GetLabelByID(ctx, labelID) - if err != nil { - if IsErrLabelNotExist(err) { + if l.BelongsToOrg() && l.OrgID != id { + return nil + } + if l.BelongsToRepo() && l.RepoID != id { return nil } - return err - } - - sess := db.GetEngine(ctx) - - if l.BelongsToOrg() && l.OrgID != id { - return nil - } - if l.BelongsToRepo() && l.RepoID != id { - return nil - } - if _, err = sess.ID(labelID).Delete(new(Label)); err != nil { - return err - } + if _, err = sess.ID(labelID).Delete(new(Label)); err != nil { + return err + } - if _, err = sess. - Where("label_id = ?", labelID). - Delete(new(IssueLabel)); err != nil { - return err - } + if _, err = sess. + Where("label_id = ?", labelID). + Delete(new(IssueLabel)); err != nil { + return err + } - // delete comments about now deleted label_id - if _, err = sess.Where("label_id = ?", labelID).Cols("label_id").Delete(&Comment{}); err != nil { - return err - } + // delete comments about now deleted label_id + if _, err = sess.Where("label_id = ?", labelID).Cols("label_id").Delete(&Comment{}); err != nil { + return err + } - return nil + return nil + }) } // GetLabelByID returns a label by given ID. @@ -466,13 +450,13 @@ func UpdateLabelsByRepoID(repoID int64, labels ...*Label) error { } for _, l := range labelsToUpdate { - if err = updateLabel(ctx, l); err != nil { + if err = UpdateLabel(ctx, l); err != nil { return err } } for _, l := range labelsToDelete { - if err = deleteLabel(ctx, repoID, l.ID); err != nil { + if err = DeleteLabel(ctx, repoID, l.ID); err != nil { return err } } diff --git a/models/issues/label_test.go b/models/issues/label_test.go index 3f0e980b318be..eacaaae771e26 100644 --- a/models/issues/label_test.go +++ b/models/issues/label_test.go @@ -269,7 +269,7 @@ func TestUpdateLabel(t *testing.T) { } label.Color = update.Color label.Name = update.Name - assert.NoError(t, issues_model.UpdateLabel(update)) + assert.NoError(t, issues_model.UpdateLabel(db.DefaultContext, update)) newLabel := unittest.AssertExistsAndLoadBean(t, &issues_model.Label{ID: 1}) assert.EqualValues(t, label.ID, newLabel.ID) assert.EqualValues(t, label.Color, newLabel.Color) @@ -282,13 +282,13 @@ func TestUpdateLabel(t *testing.T) { func TestDeleteLabel(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) label := unittest.AssertExistsAndLoadBean(t, &issues_model.Label{ID: 1}) - assert.NoError(t, issues_model.DeleteLabel(label.RepoID, label.ID)) + assert.NoError(t, issues_model.DeleteLabel(db.DefaultContext, label.RepoID, label.ID)) unittest.AssertNotExistsBean(t, &issues_model.Label{ID: label.ID, RepoID: label.RepoID}) - assert.NoError(t, issues_model.DeleteLabel(label.RepoID, label.ID)) + assert.NoError(t, issues_model.DeleteLabel(db.DefaultContext, label.RepoID, label.ID)) unittest.AssertNotExistsBean(t, &issues_model.Label{ID: label.ID}) - assert.NoError(t, issues_model.DeleteLabel(unittest.NonexistentID, unittest.NonexistentID)) + assert.NoError(t, issues_model.DeleteLabel(db.DefaultContext, unittest.NonexistentID, unittest.NonexistentID)) unittest.CheckConsistencyFor(t, &issues_model.Label{}, &repo_model.Repository{}) } diff --git a/routers/api/v1/org/label.go b/routers/api/v1/org/label.go index 9ef28d4db9267..3e82d5b3a3a10 100644 --- a/routers/api/v1/org/label.go +++ b/routers/api/v1/org/label.go @@ -210,7 +210,7 @@ func EditLabel(ctx *context.APIContext) { l.Description = *form.Description } l.SetArchived(form.IsArchived != nil && *form.IsArchived) - if err := issues_model.UpdateLabel(l); err != nil { + if err := issues_model.UpdateLabel(ctx, l); err != nil { ctx.Error(http.StatusInternalServerError, "UpdateLabel", err) return } @@ -239,7 +239,7 @@ func DeleteLabel(ctx *context.APIContext) { // "204": // "$ref": "#/responses/empty" - if err := issues_model.DeleteLabel(ctx.Org.Organization.ID, ctx.ParamsInt64(":id")); err != nil { + if err := issues_model.DeleteLabel(ctx, ctx.Org.Organization.ID, ctx.ParamsInt64(":id")); err != nil { ctx.Error(http.StatusInternalServerError, "DeleteLabel", err) return } diff --git a/routers/api/v1/repo/label.go b/routers/api/v1/repo/label.go index fc9a16b58aa89..1083eb2005736 100644 --- a/routers/api/v1/repo/label.go +++ b/routers/api/v1/repo/label.go @@ -232,7 +232,7 @@ func EditLabel(ctx *context.APIContext) { l.Description = *form.Description } l.SetArchived(form.IsArchived != nil && *form.IsArchived) - if err := issues_model.UpdateLabel(l); err != nil { + if err := issues_model.UpdateLabel(ctx, l); err != nil { ctx.Error(http.StatusInternalServerError, "UpdateLabel", err) return } @@ -266,7 +266,7 @@ func DeleteLabel(ctx *context.APIContext) { // "204": // "$ref": "#/responses/empty" - if err := issues_model.DeleteLabel(ctx.Repo.Repository.ID, ctx.ParamsInt64(":id")); err != nil { + if err := issues_model.DeleteLabel(ctx, ctx.Repo.Repository.ID, ctx.ParamsInt64(":id")); err != nil { ctx.Error(http.StatusInternalServerError, "DeleteLabel", err) return } diff --git a/routers/web/org/org_labels.go b/routers/web/org/org_labels.go index 2c7725e38da5c..f78bd00274263 100644 --- a/routers/web/org/org_labels.go +++ b/routers/web/org/org_labels.go @@ -76,7 +76,7 @@ func UpdateLabel(ctx *context.Context) { l.Description = form.Description l.Color = form.Color l.SetArchived(form.IsArchived) - if err := issues_model.UpdateLabel(l); err != nil { + if err := issues_model.UpdateLabel(ctx, l); err != nil { ctx.ServerError("UpdateLabel", err) return } @@ -85,7 +85,7 @@ func UpdateLabel(ctx *context.Context) { // DeleteLabel delete a label func DeleteLabel(ctx *context.Context) { - if err := issues_model.DeleteLabel(ctx.Org.Organization.ID, ctx.FormInt64("id")); err != nil { + if err := issues_model.DeleteLabel(ctx, ctx.Org.Organization.ID, ctx.FormInt64("id")); err != nil { ctx.Flash.Error("DeleteLabel: " + err.Error()) } else { ctx.Flash.Success(ctx.Tr("repo.issues.label_deletion_success")) diff --git a/routers/web/repo/issue_label.go b/routers/web/repo/issue_label.go index 257610d3af54b..e994c2dc47624 100644 --- a/routers/web/repo/issue_label.go +++ b/routers/web/repo/issue_label.go @@ -145,7 +145,7 @@ func UpdateLabel(ctx *context.Context) { l.Color = form.Color l.SetArchived(form.IsArchived) - if err := issues_model.UpdateLabel(l); err != nil { + if err := issues_model.UpdateLabel(ctx, l); err != nil { ctx.ServerError("UpdateLabel", err) return } @@ -154,7 +154,7 @@ func UpdateLabel(ctx *context.Context) { // DeleteLabel delete a label func DeleteLabel(ctx *context.Context) { - if err := issues_model.DeleteLabel(ctx.Repo.Repository.ID, ctx.FormInt64("id")); err != nil { + if err := issues_model.DeleteLabel(ctx, ctx.Repo.Repository.ID, ctx.FormInt64("id")); err != nil { ctx.Flash.Error("DeleteLabel: " + err.Error()) } else { ctx.Flash.Success(ctx.Tr("repo.issues.label_deletion_success")) From 35ab3bb1d2fe146d2bda308716ab8d3f7e0a4de4 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 24 Aug 2023 01:09:52 -0400 Subject: [PATCH 75/94] No magic literal --- web_src/js/features/repo-migration.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web_src/js/features/repo-migration.js b/web_src/js/features/repo-migration.js index 074d27332f0e6..0fa4c907a1b9f 100644 --- a/web_src/js/features/repo-migration.js +++ b/web_src/js/features/repo-migration.js @@ -11,6 +11,9 @@ const $lfsSettings = $('#lfs_settings'); const $lfsEndpoint = $('#lfs_endpoint'); const $items = $('#migrate_items').find('input[type=checkbox]'); +// services that supports migration items with mirroring +const allowedServiceTypes = [2]; // 2 = GitHub service type + export function initRepoMigration() { checkAuth(); setLFSSettingsVisibility(); @@ -45,8 +48,7 @@ function checkItems(tokenAuth) { enableItems = $user.val() !== '' || $pass.val() !== ''; } if (enableItems && $service.val() > 1) { - const allowedServices = [2]; // services that supports migration items with mirroring - if ($mirror.is(':checked') && !allowedServices.includes(Number($service.val()))) { + if ($mirror.is(':checked') && !allowedServiceTypes.includes(Number($service.val()))) { $items.not('[name="wiki"]').attr('disabled', true); $items.filter('[name="wiki"]').attr('disabled', false); return; From 4d2cb07cd45d23b25733ff73b622e43cdbb55aa9 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 24 Aug 2023 13:15:17 -0400 Subject: [PATCH 76/94] Add comments for OriginalID --- models/issues/review.go | 2 +- modules/migration/review.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/issues/review.go b/models/issues/review.go index 67aac76ff986c..dcd019e781f30 100644 --- a/models/issues/review.go +++ b/models/issues/review.go @@ -111,7 +111,7 @@ type Review struct { ReviewerTeam *organization.Team `xorm:"-"` OriginalAuthor string OriginalAuthorID int64 - OriginalID int64 + OriginalID int64 // ID from the upstream syncing source Issue *Issue `xorm:"-"` IssueID int64 `xorm:"index"` Content string `xorm:"TEXT"` diff --git a/modules/migration/review.go b/modules/migration/review.go index a778f84e54f9b..a9a2a411aba47 100644 --- a/modules/migration/review.go +++ b/modules/migration/review.go @@ -42,7 +42,7 @@ type Review struct { CreatedAt time.Time `yaml:"created_at"` State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT Comments []*ReviewComment - OriginalID int64 + OriginalID int64 // ID from the upstream syncing source } // GetExternalName ExternalUserMigrated interface @@ -65,5 +65,5 @@ type ReviewComment struct { Reactions []*Reaction CreatedAt time.Time `yaml:"created_at"` UpdatedAt time.Time `yaml:"updated_at"` - OriginalID int64 + OriginalID int64 // ID from the upstream syncing source } From 6962c14cf321ac027289d2da3d33695e4bb64cee Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 24 Aug 2023 13:23:01 -0400 Subject: [PATCH 77/94] Update log text --- services/mirror/mirror_pull.go | 46 ++++++++++++++++------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 1f6292d758766..d6bc23d24c0f7 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -455,28 +455,26 @@ func runSyncMisc(ctx context.Context, m *repo_model.Mirror, lastSynced time.Time _, err := migrations.SyncRepository(ctx, m.Repo.MustOwner(ctx), repo, opts, nil, lastSynced) if err != nil { - log.Error("SyncMirrors [repo: %-v]: failed to sync repository: %v", m.Repo, err) - return false + log.Error("runSyncMisc [repo: %-v]: failed to run SyncRepository: %v", m.Repo, err) } - - return true + return err == nil } // SyncPullMirror starts the sync of the pull mirror and schedules the next run. func SyncPullMirror(ctx context.Context, repoID int64) bool { - log.Trace("SyncMirrors [repo_id: %v]", repoID) + log.Trace("SyncPullMirror [repo_id: %v]", repoID) defer func() { err := recover() if err == nil { return } // There was a panic whilst syncMirrors... - log.Error("PANIC whilst SyncMirrors[repo_id: %d] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2)) + log.Error("PANIC whilst SyncPullMirror[repo_id: %d] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2)) }() m, err := repo_model.GetMirrorByRepoID(ctx, repoID) if err != nil { - log.Error("SyncMirrors [repo_id: %v]: unable to GetMirrorByRepoID: %v", repoID, err) + log.Error("SyncPullMirror [repo_id: %v]: unable to GetMirrorByRepoID: %v", repoID, err) return false } _ = m.GetRepository() // force load repository of mirror @@ -487,37 +485,37 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Syncing Mirror %s/%s", m.Repo.OwnerName, m.Repo.Name)) defer finished() - log.Trace("SyncMirrors [repo: %-v]: Running Sync Git", m.Repo) + log.Trace("SyncPullMirror [repo: %-v]: Running Sync Git", m.Repo) results, ok := runSyncGit(ctx, m) if !ok { if err = repo_model.TouchMirror(ctx, m); err != nil { - log.Error("SyncMirrors [repo: %-v]: failed to TouchMirror: %v", m.Repo, err) + log.Error("SyncPullMirror [repo: %-v]: failed to TouchMirror: %v", m.Repo, err) } return false } if ok := runSyncMisc(ctx, m, lastSynced); !ok { if err = repo_model.TouchMirror(ctx, m); err != nil { - log.Error("SyncMirrors [repo: %-v]: failed to TouchMirror: %v", m.Repo, err) + log.Error("SyncPullMirror [repo: %-v]: failed to TouchMirror: %v", m.Repo, err) } return false } - log.Trace("SyncMirrors [repo: %-v]: Scheduling next update", m.Repo) + log.Trace("SyncPullMirror [repo: %-v]: Scheduling next update", m.Repo) m.ScheduleNextUpdate() if err = repo_model.UpdateMirror(ctx, m); err != nil { - log.Error("SyncMirrors [repo: %-v]: failed to UpdateMirror with next update date: %v", m.Repo, err) + log.Error("SyncPullMirror [repo: %-v]: failed to UpdateMirror with next update date: %v", m.Repo, err) return false } var gitRepo *git.Repository if len(results) == 0 { - log.Trace("SyncMirrors [repo: %-v]: no branches updated", m.Repo) + log.Trace("SyncPullMirror [repo: %-v]: no branches updated", m.Repo) } else { - log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results)) + log.Trace("SyncPullMirror [repo: %-v]: %d branches updated", m.Repo, len(results)) gitRepo, err = git.OpenRepository(ctx, m.Repo.RepoPath()) if err != nil { - log.Error("SyncMirrors [repo: %-v]: unable to OpenRepository: %v", m.Repo, err) + log.Error("SyncPullMirror [repo: %-v]: unable to OpenRepository: %v", m.Repo, err) return false } defer gitRepo.Close() @@ -537,7 +535,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { if result.oldCommitID == gitShortEmptySha { commitID, err := gitRepo.GetRefCommitID(result.refName.String()) if err != nil { - log.Error("SyncMirrors [repo: %-v]: unable to GetRefCommitID [ref_name: %s]: %v", m.Repo, result.refName, err) + log.Error("SyncPullMirror [repo: %-v]: unable to GetRefCommitID [ref_name: %s]: %v", m.Repo, result.refName, err) continue } notification.NotifySyncPushCommits(ctx, m.Repo.MustOwner(ctx), m.Repo, &repo_module.PushUpdateOptions{ @@ -558,17 +556,17 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { // Push commits oldCommitID, err := git.GetFullCommitID(gitRepo.Ctx, gitRepo.Path, result.oldCommitID) if err != nil { - log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID[%s]: %v", m.Repo, result.oldCommitID, err) + log.Error("SyncPullMirror [repo: %-v]: unable to get GetFullCommitID[%s]: %v", m.Repo, result.oldCommitID, err) continue } newCommitID, err := git.GetFullCommitID(gitRepo.Ctx, gitRepo.Path, result.newCommitID) if err != nil { - log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID [%s]: %v", m.Repo, result.newCommitID, err) + log.Error("SyncPullMirror [repo: %-v]: unable to get GetFullCommitID [%s]: %v", m.Repo, result.newCommitID, err) continue } commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID) if err != nil { - log.Error("SyncMirrors [repo: %-v]: unable to get CommitsBetweenIDs [new_commit_id: %s, old_commit_id: %s]: %v", m.Repo, newCommitID, oldCommitID, err) + log.Error("SyncPullMirror [repo: %-v]: unable to get CommitsBetweenIDs [new_commit_id: %s, old_commit_id: %s]: %v", m.Repo, newCommitID, oldCommitID, err) continue } @@ -578,7 +576,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { } if newCommit, err := gitRepo.GetCommit(newCommitID); err != nil { - log.Error("SyncMirrors [repo: %-v]: unable to get commit %s: %v", m.Repo, newCommitID, err) + log.Error("SyncPullMirror [repo: %-v]: unable to get commit %s: %v", m.Repo, newCommitID, err) continue } else { theCommits.HeadCommit = repo_module.CommitToPushCommit(newCommit) @@ -592,21 +590,21 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { NewCommitID: newCommitID, }, theCommits) } - log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo) + log.Trace("SyncPullMirror [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo) // Get latest commit date and update to current repository updated time commitDate, err := git.GetLatestCommitTime(ctx, m.Repo.RepoPath()) if err != nil { - log.Error("SyncMirrors [repo: %-v]: unable to GetLatestCommitDate: %v", m.Repo, err) + log.Error("SyncPullMirror [repo: %-v]: unable to GetLatestCommitDate: %v", m.Repo, err) return false } if err = repo_model.UpdateRepositoryUpdatedTime(m.RepoID, commitDate); err != nil { - log.Error("SyncMirrors [repo: %-v]: unable to update repository 'updated_unix': %v", m.Repo, err) + log.Error("SyncPullMirror [repo: %-v]: unable to update repository 'updated_unix': %v", m.Repo, err) return false } - log.Trace("SyncMirrors [repo: %-v]: Successfully updated", m.Repo) + log.Trace("SyncPullMirror [repo: %-v]: Successfully updated", m.Repo) return true } From 4a33b2a1b377186a172e527464f19ec8f6de9861 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 24 Aug 2023 13:47:08 -0400 Subject: [PATCH 78/94] Revert sync wiki setting check --- services/mirror/mirror_pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index d6bc23d24c0f7..eabcb68421802 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -335,7 +335,7 @@ func runSyncGit(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, log.Error("SyncMirrors [repo: %-v]: failed to update size for mirror repository: %v", m.Repo, err) } - if m.Repo.HasWiki() && m.SyncWiki { + if m.Repo.HasWiki() { log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo) stderrBuilder.Reset() stdoutBuilder.Reset() From 884cb6d3502096e2485fb3d4259a3960a9f744dd Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 24 Aug 2023 13:52:01 -0400 Subject: [PATCH 79/94] Default sync items setting to false --- models/repo/mirror.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/models/repo/mirror.go b/models/repo/mirror.go index 03a75373b8774..83ba1814f9615 100644 --- a/models/repo/mirror.go +++ b/models/repo/mirror.go @@ -25,13 +25,13 @@ type Mirror struct { Interval time.Duration EnablePrune bool `xorm:"NOT NULL DEFAULT true"` - SyncWiki bool - SyncIssues bool - SyncMilestones bool - SyncLabels bool - SyncReleases bool - SyncComments bool - SyncPullRequests bool + SyncWiki bool `xorm:"NOT NULL DEFAULT false"` + SyncIssues bool `xorm:"NOT NULL DEFAULT false"` + SyncMilestones bool `xorm:"NOT NULL DEFAULT false"` + SyncLabels bool `xorm:"NOT NULL DEFAULT false"` + SyncReleases bool `xorm:"NOT NULL DEFAULT false"` + SyncComments bool `xorm:"NOT NULL DEFAULT false"` + SyncPullRequests bool `xorm:"NOT NULL DEFAULT false"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` From fa40aa7093d9ff54d5a64b7cacbf9834072f267a Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 24 Aug 2023 14:47:31 -0400 Subject: [PATCH 80/94] Expose one more context --- models/issues/label.go | 103 ++++++++++++-------------- services/migrations/gitea_uploader.go | 3 +- 2 files changed, 51 insertions(+), 55 deletions(-) diff --git a/models/issues/label.go b/models/issues/label.go index 98fdd7b8146e9..edf7b7ce300bd 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -397,72 +397,67 @@ func GetLabelsByRepoID(ctx context.Context, repoID int64, sortType string, listO return labels, sess.Find(&labels) } -// UpdateLabels adds, updates, and deletes relevant labels for the given repository. -func UpdateLabelsByRepoID(repoID int64, labels ...*Label) error { - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - - existingLabels, err := GetLabelsByRepoID(ctx, repoID, "", db.ListOptions{}) - if err != nil { - return err - } - labelsToAdd := make([]*Label, 0) - labelsToUpdate := make([]*Label, 0) - labelsToDelete := make([]*Label, 0) - - for _, l := range labels { - var foundLabel *Label - for _, existingLabel := range existingLabels { - if existingLabel.OriginalID == l.OriginalID { - foundLabel = existingLabel - break - } +// UpdateLabelsByRepoID adds, updates, and deletes relevant labels for the given repository. +func UpdateLabelsByRepoID(ctx context.Context, repoID int64, labels ...*Label) error { + return db.WithTx(ctx, func(ctx context.Context) error { + existingLabels, err := GetLabelsByRepoID(ctx, repoID, "", db.ListOptions{}) + if err != nil { + return err } + labelsToAdd := make([]*Label, 0) + labelsToUpdate := make([]*Label, 0) + labelsToDelete := make([]*Label, 0) + + for _, l := range labels { + var foundLabel *Label + for _, existingLabel := range existingLabels { + if existingLabel.OriginalID == l.OriginalID { + foundLabel = existingLabel + break + } + } - if foundLabel == nil { - labelsToAdd = append(labelsToAdd, l) - } else if foundLabel.Name != l.Name || foundLabel.Description != l.Description || - foundLabel.Color != l.Color { - l.RepoID = repoID - labelsToUpdate = append(labelsToUpdate, l) + if foundLabel == nil { + labelsToAdd = append(labelsToAdd, l) + } else if foundLabel.Name != l.Name || foundLabel.Description != l.Description || + foundLabel.Color != l.Color { + l.RepoID = repoID + labelsToUpdate = append(labelsToUpdate, l) + } } - } - for _, existingLabel := range existingLabels { - found := false - for _, label := range labels { - if label.OriginalID == existingLabel.OriginalID { - found = true - break + for _, existingLabel := range existingLabels { + found := false + for _, label := range labels { + if label.OriginalID == existingLabel.OriginalID { + found = true + break + } + } + if !found { + labelsToDelete = append(labelsToDelete, existingLabel) } } - if !found { - labelsToDelete = append(labelsToDelete, existingLabel) - } - } - for _, l := range labelsToAdd { - if err = NewLabel(ctx, l); err != nil { - return err + for _, l := range labelsToAdd { + if err = NewLabel(ctx, l); err != nil { + return err + } } - } - for _, l := range labelsToUpdate { - if err = UpdateLabel(ctx, l); err != nil { - return err + for _, l := range labelsToUpdate { + if err = UpdateLabel(ctx, l); err != nil { + return err + } } - } - for _, l := range labelsToDelete { - if err = DeleteLabel(ctx, repoID, l.ID); err != nil { - return err + for _, l := range labelsToDelete { + if err = DeleteLabel(ctx, repoID, l.ID); err != nil { + return err + } } - } - - return committer.Commit() + return nil + }) } // CountLabelsByRepoID count number of all labels that belong to given repository by ID. diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index ccd84098506b1..6fcc44f2a8b9a 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -1009,7 +1009,8 @@ func (g *GiteaLocalUploader) UpdateMilestones(milestones ...*base.Milestone) err func (g *GiteaLocalUploader) UpdateLabels(labels ...*base.Label) error { lbs := g.convertLabels(labels...) - if err := issues_model.UpdateLabelsByRepoID(g.repo.ID, lbs...); err != nil { + ctx := db.DefaultContext + if err := issues_model.UpdateLabelsByRepoID(ctx, g.repo.ID, lbs...); err != nil { return err } for _, lb := range lbs { From 2a66750c0d3f42b49e3d79ff3ca0938b5bdca553 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 24 Aug 2023 14:48:23 -0400 Subject: [PATCH 81/94] Update comment --- modules/migration/uploader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/migration/uploader.go b/modules/migration/uploader.go index 97e2118689542..225a74340c5e6 100644 --- a/modules/migration/uploader.go +++ b/modules/migration/uploader.go @@ -17,9 +17,9 @@ type Uploader interface { CreateComments(comments ...*Comment) error CreatePullRequests(prs ...*PullRequest) error CreateReviews(reviews ...*Review) error - UpdateTopics(topic ...string) error // update topics of a repository, and delete unused ones - UpdateMilestones(milestones ...*Milestone) error // update milestones of a repository, and delete unused ones - UpdateLabels(labels ...*Label) error // rewrite all issue labels and delete unused ones + UpdateTopics(topic ...string) error // update topics of a repository, and delete those that are not in the list + UpdateMilestones(milestones ...*Milestone) error // update milestones of a repository, and delete those that are not in the list + UpdateLabels(labels ...*Label) error // rewrite all issue labels and delete those that are not in the list PatchReleases(releases ...*Release) error // add or update releases (no deletes) PatchComments(comments ...*Comment) error // add or update comments (no deletes) PatchIssues(issues ...*Issue) error // add or update issues (no deletes) From d2ef14c17f00dbf2d8f906c7f746ea36bfb40114 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 24 Aug 2023 15:01:01 -0400 Subject: [PATCH 82/94] Plural --- modules/migration/uploader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/migration/uploader.go b/modules/migration/uploader.go index 225a74340c5e6..f1e4c1cb6cc71 100644 --- a/modules/migration/uploader.go +++ b/modules/migration/uploader.go @@ -8,7 +8,7 @@ package migration type Uploader interface { MaxBatchInsertSize(tp string) int CreateRepo(repo *Repository, opts MigrateOptions) error - CreateTopics(topic ...string) error + CreateTopics(topics ...string) error CreateMilestones(milestones ...*Milestone) error CreateReleases(releases ...*Release) error SyncTags() error @@ -17,7 +17,7 @@ type Uploader interface { CreateComments(comments ...*Comment) error CreatePullRequests(prs ...*PullRequest) error CreateReviews(reviews ...*Review) error - UpdateTopics(topic ...string) error // update topics of a repository, and delete those that are not in the list + UpdateTopics(topics ...string) error // update topics of a repository, and delete those that are not in the list UpdateMilestones(milestones ...*Milestone) error // update milestones of a repository, and delete those that are not in the list UpdateLabels(labels ...*Label) error // rewrite all issue labels and delete those that are not in the list PatchReleases(releases ...*Release) error // add or update releases (no deletes) From 9bfd3e309ee3062c2d5cd64debc8c2396ee5dbdd Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 24 Aug 2023 15:15:44 -0400 Subject: [PATCH 83/94] Expose more context --- models/issues/review.go | 114 +++++------ models/migrate.go | 279 ++++++++++++-------------- services/migrations/gitea_uploader.go | 14 +- 3 files changed, 196 insertions(+), 211 deletions(-) diff --git a/models/issues/review.go b/models/issues/review.go index dcd019e781f30..18f5aad2eb694 100644 --- a/models/issues/review.go +++ b/models/issues/review.go @@ -552,90 +552,86 @@ func InsertReviews(reviews []*Review) error { // UpsertReviews inserts new reviews and updates existing ones. // This function is used for syncing from the pull mirror. -func UpsertReviews(reviews []*Review) error { - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - sess := db.GetEngine(ctx) - - for _, review := range reviews { - exists, err := sess.Where("original_id = ?", review.OriginalID).Exist(&Review{}) - if err != nil { - return err - } - - if !exists { - if _, err := sess.NoAutoTime().Insert(review); err != nil { - return err - } +func UpsertReviews(ctx context.Context, reviews []*Review) error { + return db.WithTx(ctx, func(ctx context.Context) error { + sess := db.GetEngine(ctx) - if _, err := sess.NoAutoTime().Insert(generateCommentFromReview(review)); err != nil { + for _, review := range reviews { + exists, err := sess.Where("original_id = ?", review.OriginalID).Exist(&Review{}) + if err != nil { return err } - for _, c := range review.Comments { - c.ReviewID = review.ID - } - - if len(review.Comments) > 0 { - if _, err := sess.NoAutoTime().Insert(review.Comments); err != nil { + if !exists { + if _, err := sess.NoAutoTime().Insert(review); err != nil { return err } - } - } else { - if _, err = sess.NoAutoTime().Where("original_id = ?", review.OriginalID).Update(review); err != nil { - return err - } - // Get id of the review - if err = sess.NoAutoTime().Where("original_id = ?", review.OriginalID).Find(review); err != nil { - return err - } + if _, err := sess.NoAutoTime().Insert(generateCommentFromReview(review)); err != nil { + return err + } - comment := generateCommentFromReview(review) - exists, err := existsCommentByReviewIDAndCreatedUnix(sess, comment) - if err != nil { - return err - } + for _, c := range review.Comments { + c.ReviewID = review.ID + } - if !exists { - if _, err := sess.NoAutoTime().Insert(comment); err != nil { - return err + if len(review.Comments) > 0 { + if _, err := sess.NoAutoTime().Insert(review.Comments); err != nil { + return err + } } } else { - if _, err := sess.NoAutoTime().Where("original_id = ?", comment.OriginalID).Update(comment); err != nil { + if _, err = sess.NoAutoTime().Where("original_id = ?", review.OriginalID).Update(review); err != nil { return err } - } - for _, c := range review.Comments { - c.ReviewID = review.ID - } + // Get id of the review + if err = sess.NoAutoTime().Where("original_id = ?", review.OriginalID).Find(review); err != nil { + return err + } + + comment := generateCommentFromReview(review) + exists, err := existsCommentByReviewIDAndCreatedUnix(sess, comment) + if err != nil { + return err + } - if len(review.Comments) > 0 { - for _, comment := range review.Comments { - exists, err := existsCommentByReviewIDAndCreatedUnix(sess, comment) - if err != nil { + if !exists { + if _, err := sess.NoAutoTime().Insert(comment); err != nil { + return err + } + } else { + if _, err := sess.NoAutoTime().Where("original_id = ?", comment.OriginalID).Update(comment); err != nil { return err } + } - if !exists { - if _, err := sess.NoAutoTime().Insert(comment); err != nil { + for _, c := range review.Comments { + c.ReviewID = review.ID + } + + if len(review.Comments) > 0 { + for _, comment := range review.Comments { + exists, err := existsCommentByReviewIDAndCreatedUnix(sess, comment) + if err != nil { return err } - } else { - if _, err := sess.NoAutoTime().Where("original_id = ?", comment.OriginalID).Update(comment); err != nil { - return err + + if !exists { + if _, err := sess.NoAutoTime().Insert(comment); err != nil { + return err + } + } else { + if _, err := sess.NoAutoTime().Where("original_id = ?", comment.OriginalID).Update(comment); err != nil { + return err + } } } } } } - } - - return committer.Commit() + return nil + }) } func existsCommentByReviewIDAndCreatedUnix(sess db.Engine, comment *Comment) (bool, error) { diff --git a/models/migrate.go b/models/migrate.go index 125970b1d4de4..3afcd3338284b 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -41,89 +41,85 @@ func InsertMilestones(ms ...*issues_model.Milestone) (err error) { } // UpdateMilestones updates milestones of repository. -func UpdateMilestones(ms ...*issues_model.Milestone) (err error) { +func UpdateMilestones(ctx context.Context, ms ...*issues_model.Milestone) (err error) { if len(ms) == 0 { return nil } - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - sess := db.GetEngine(ctx) + return db.WithTx(ctx, func(ctx context.Context) error { + sess := db.GetEngine(ctx) - // get existing milestones - existingMilestones := make([]*issues_model.Milestone, 0) - if err = sess.Where("repo_id = ?", ms[0].RepoID).Find(&existingMilestones); err != nil { - return err - } + // get existing milestones + existingMilestones := make([]*issues_model.Milestone, 0) + if err = sess.Where("repo_id = ?", ms[0].RepoID).Find(&existingMilestones); err != nil { + return err + } - milestonesToAdd := make([]*issues_model.Milestone, 0) - milestonesToUpdate := make([]*issues_model.Milestone, 0) - milestonesToDelete := make([]*issues_model.Milestone, 0) - foundMap := make(map[int64]bool) + milestonesToAdd := make([]*issues_model.Milestone, 0) + milestonesToUpdate := make([]*issues_model.Milestone, 0) + milestonesToDelete := make([]*issues_model.Milestone, 0) + foundMap := make(map[int64]bool) - openCount := 0 - closedCount := 0 + openCount := 0 + closedCount := 0 - for _, m := range ms { - var foundMilestone *issues_model.Milestone - for _, existingMilestone := range existingMilestones { - if existingMilestone.OriginalID == m.OriginalID { - foundMilestone = existingMilestone - foundMap[existingMilestone.ID] = true - break + for _, m := range ms { + var foundMilestone *issues_model.Milestone + for _, existingMilestone := range existingMilestones { + if existingMilestone.OriginalID == m.OriginalID { + foundMilestone = existingMilestone + foundMap[existingMilestone.ID] = true + break + } + } + + if foundMilestone == nil { + milestonesToAdd = append(milestonesToAdd, m) + } else if foundMilestone.OriginalID != m.OriginalID { + m.ID = foundMilestone.ID + milestonesToUpdate = append(milestonesToUpdate, m) } - } - if foundMilestone == nil { - milestonesToAdd = append(milestonesToAdd, m) - } else if foundMilestone.OriginalID != m.OriginalID { - m.ID = foundMilestone.ID - milestonesToUpdate = append(milestonesToUpdate, m) + if m.IsClosed { + closedCount++ + } else { + openCount++ + } } - if m.IsClosed { - closedCount++ - } else { - openCount++ + for _, existingMilestone := range existingMilestones { + if _, exist := foundMap[existingMilestone.ID]; !exist { + milestonesToDelete = append(milestonesToDelete, existingMilestone) + } } - } - for _, existingMilestone := range existingMilestones { - if _, exist := foundMap[existingMilestone.ID]; !exist { - milestonesToDelete = append(milestonesToDelete, existingMilestone) + if len(milestonesToAdd) > 0 { + if _, err = sess.Insert(milestonesToAdd); err != nil { + return err + } } - } - if len(milestonesToAdd) > 0 { - if _, err = sess.Insert(milestonesToAdd); err != nil { - return err + for _, m := range milestonesToUpdate { + if _, err = sess.ID(m.ID).AllCols().Update(m); err != nil { + return err + } } - } - for _, m := range milestonesToUpdate { - if _, err = sess.ID(m.ID).AllCols().Update(m); err != nil { - return err + for _, m := range milestonesToDelete { + if _, err = sess.ID(m.ID).Delete(m); err != nil { + return err + } } - } - for _, m := range milestonesToDelete { - if _, err = sess.ID(m.ID).Delete(m); err != nil { + if _, err = sess.ID(ms[0].RepoID).Update(&repo_model.Repository{ + NumMilestones: len(ms), + NumOpenMilestones: openCount, + NumClosedMilestones: closedCount, + }); err != nil { return err } - } - - if _, err = sess.ID(ms[0].RepoID).Update(&repo_model.Repository{ - NumMilestones: len(ms), - NumOpenMilestones: openCount, - NumClosedMilestones: closedCount, - }); err != nil { - return err - } - - return committer.Commit() + return nil + }) } // InsertIssues insert issues to database @@ -179,19 +175,15 @@ func insertIssue(ctx context.Context, issue *issues_model.Issue) error { } // UpsertIssues creates new issues and updates existing issues in database -func UpsertIssues(issues ...*issues_model.Issue) error { - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - - for _, issue := range issues { - if _, err := upsertIssue(ctx, issue); err != nil { - return err +func UpsertIssues(ctx context.Context, issues ...*issues_model.Issue) error { + return db.WithTx(ctx, func(ctx context.Context) error { + for _, issue := range issues { + if _, err := upsertIssue(ctx, issue); err != nil { + return err + } } - } - return committer.Commit() + return nil + }) } func updateIssue(ctx context.Context, issue *issues_model.Issue) error { @@ -296,7 +288,7 @@ func InsertIssueComments(comments []*issues_model.Comment) error { } // UpsertIssueComments inserts many comments of issues. -func UpsertIssueComments(comments []*issues_model.Comment) error { +func UpsertIssueComments(ctx context.Context, comments []*issues_model.Comment) error { if len(comments) == 0 { return nil } @@ -306,71 +298,67 @@ func UpsertIssueComments(comments []*issues_model.Comment) error { issueIDs[comment.IssueID] = true } - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - - sess := db.GetEngine(ctx) - for _, comment := range comments { - exists, err := sess.Exist(&issues_model.Comment{ - IssueID: comment.IssueID, - CreatedUnix: comment.CreatedUnix, - }) - if err != nil { - return err - } - if !exists { - if _, err := sess.NoAutoTime().Insert(comment); err != nil { + return db.WithTx(ctx, func(ctx context.Context) error { + sess := db.GetEngine(ctx) + for _, comment := range comments { + exists, err := sess.Exist(&issues_model.Comment{ + IssueID: comment.IssueID, + CreatedUnix: comment.CreatedUnix, + }) + if err != nil { return err } - } else { - if _, err := sess.NoAutoTime().Where( - "original_id = ?", comment.IssueID, comment.OriginalID, - ).AllCols().Update(comment); err != nil { - return err + if !exists { + if _, err := sess.NoAutoTime().Insert(comment); err != nil { + return err + } + } else { + if _, err := sess.NoAutoTime().Where( + "original_id = ?", comment.IssueID, comment.OriginalID, + ).AllCols().Update(comment); err != nil { + return err + } } - } - for _, reaction := range comment.Reactions { - reaction.IssueID = comment.IssueID - reaction.CommentID = comment.ID - } - if len(comment.Reactions) > 0 { for _, reaction := range comment.Reactions { - // issue is uniquely identified by issue_id, comment_id and type - exists, err := sess.Exist(&issues_model.Reaction{ - IssueID: reaction.IssueID, - CommentID: reaction.CommentID, - Type: reaction.Type, - }) - if err != nil { - return err - } - if exists { - if _, err := sess.Where( - "issue_id = ? AND comment_id = ? AND type = ?", - reaction.IssueID, reaction.CommentID, reaction.Type, - ).AllCols().Update(&reaction); err != nil { + reaction.IssueID = comment.IssueID + reaction.CommentID = comment.ID + } + if len(comment.Reactions) > 0 { + for _, reaction := range comment.Reactions { + // issue comment rection is uniquely identified by issue_id, comment_id and type + exists, err := sess.Exist(&issues_model.Reaction{ + IssueID: reaction.IssueID, + CommentID: reaction.CommentID, + Type: reaction.Type, + }) + if err != nil { return err } - } else { - if _, err := sess.Insert(&reaction); err != nil { - return err + if exists { + if _, err := sess.Where( + "issue_id = ? AND comment_id = ? AND type = ?", + reaction.IssueID, reaction.CommentID, reaction.Type, + ).AllCols().Update(&reaction); err != nil { + return err + } + } else { + if _, err := sess.Insert(&reaction); err != nil { + return err + } } } } } - } - for issueID := range issueIDs { - if _, err := db.Exec(ctx, "UPDATE issue SET num_comments = (SELECT count(*) FROM comment WHERE issue_id = ? AND `type`=?) WHERE id = ?", - issueID, issues_model.CommentTypeComment, issueID); err != nil { - return err + for issueID := range issueIDs { + if _, err := db.Exec(ctx, "UPDATE issue SET num_comments = (SELECT count(*) FROM comment WHERE issue_id = ? AND `type`=?) WHERE id = ?", + issueID, issues_model.CommentTypeComment, issueID); err != nil { + return err + } } - } - return committer.Commit() + return nil + }) } // InsertPullRequests inserted pull requests @@ -394,31 +382,28 @@ func InsertPullRequests(ctx context.Context, prs ...*issues_model.PullRequest) e } // UpsertPullRequests inserts new pull requests and updates existing pull requests in database -func UpsertPullRequests(prs ...*issues_model.PullRequest) error { - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - sess := db.GetEngine(ctx) - for _, pr := range prs { - isInsert, err := upsertIssue(ctx, pr.Issue) - if err != nil { - return err - } - pr.IssueID = pr.Issue.ID - - if isInsert { - if _, err := sess.NoAutoTime().Insert(pr); err != nil { +func UpsertPullRequests(ctx context.Context, prs ...*issues_model.PullRequest) error { + return db.WithTx(ctx, func(ctx context.Context) error { + sess := db.GetEngine(ctx) + for _, pr := range prs { + isInsert, err := upsertIssue(ctx, pr.Issue) + if err != nil { return err } - } else { - if _, err := sess.NoAutoTime().ID(pr.ID).AllCols().Update(pr); err != nil { - return err + pr.IssueID = pr.Issue.ID + + if isInsert { + if _, err := sess.NoAutoTime().Insert(pr); err != nil { + return err + } + } else { + if _, err := sess.NoAutoTime().ID(pr.ID).AllCols().Update(pr); err != nil { + return err + } } } - } - return committer.Commit() + return nil + }) } // InsertReleases migrates release diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 6fcc44f2a8b9a..0c9223b1fa9ab 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -996,7 +996,8 @@ func (g *GiteaLocalUploader) UpdateTopics(topics ...string) error { func (g *GiteaLocalUploader) UpdateMilestones(milestones ...*base.Milestone) error { mss := g.prepareMilestones(milestones...) - err := models.UpdateMilestones(mss...) + ctx := db.DefaultContext + err := models.UpdateMilestones(ctx, mss...) if err != nil { return err } @@ -1038,7 +1039,8 @@ func (g *GiteaLocalUploader) PatchIssues(issues ...*base.Issue) error { return nil } - if err := models.UpsertIssues(iss...); err != nil { + ctx := db.DefaultContext + if err := models.UpsertIssues(ctx, iss...); err != nil { return err } @@ -1057,7 +1059,8 @@ func (g *GiteaLocalUploader) PatchComments(comments ...*base.Comment) error { if len(cms) == 0 { return nil } - return models.UpsertIssueComments(cms) + ctx := db.DefaultContext + return models.UpsertIssueComments(ctx, cms) } func (g *GiteaLocalUploader) PatchPullRequests(prs ...*base.PullRequest) error { @@ -1066,7 +1069,7 @@ func (g *GiteaLocalUploader) PatchPullRequests(prs ...*base.PullRequest) error { if err != nil { return err } - if err := models.UpsertPullRequests(gprs...); err != nil { + if err := models.UpsertPullRequests(ctx, gprs...); err != nil { return err } for _, pr := range gprs { @@ -1082,7 +1085,8 @@ func (g *GiteaLocalUploader) PatchReviews(reviews ...*base.Review) error { return err } - return issues_model.UpsertReviews(cms) + ctx := db.DefaultContext + return issues_model.UpsertReviews(ctx, cms) } // Rollback when migrating failed, this will rollback all the changes. From e39d0ff61fc1984af013ec274b763eb2772b63c0 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Thu, 24 Aug 2023 15:51:26 -0400 Subject: [PATCH 84/94] Fix a comment issue and early return for prs --- models/migrate.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/models/migrate.go b/models/migrate.go index 3afcd3338284b..3b338600c0552 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -302,8 +302,8 @@ func UpsertIssueComments(ctx context.Context, comments []*issues_model.Comment) sess := db.GetEngine(ctx) for _, comment := range comments { exists, err := sess.Exist(&issues_model.Comment{ - IssueID: comment.IssueID, - CreatedUnix: comment.CreatedUnix, + IssueID: comment.IssueID, + OriginalID: comment.OriginalID, }) if err != nil { return err @@ -383,6 +383,10 @@ func InsertPullRequests(ctx context.Context, prs ...*issues_model.PullRequest) e // UpsertPullRequests inserts new pull requests and updates existing pull requests in database func UpsertPullRequests(ctx context.Context, prs ...*issues_model.PullRequest) error { + if len(prs) == 0 { + return nil + } + return db.WithTx(ctx, func(ctx context.Context) error { sess := db.GetEngine(ctx) for _, pr := range prs { From ee776751f2134a0247ce8caa7725ee01ce1d3e42 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 25 Aug 2023 01:10:37 -0400 Subject: [PATCH 85/94] Fix comment sync error --- models/migrate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/migrate.go b/models/migrate.go index 3b338600c0552..f38b35eb82a86 100644 --- a/models/migrate.go +++ b/models/migrate.go @@ -314,7 +314,7 @@ func UpsertIssueComments(ctx context.Context, comments []*issues_model.Comment) } } else { if _, err := sess.NoAutoTime().Where( - "original_id = ?", comment.IssueID, comment.OriginalID, + "issue_id = ? AND original_id = ?", comment.IssueID, comment.OriginalID, ).AllCols().Update(comment); err != nil { return err } @@ -326,7 +326,7 @@ func UpsertIssueComments(ctx context.Context, comments []*issues_model.Comment) } if len(comment.Reactions) > 0 { for _, reaction := range comment.Reactions { - // issue comment rection is uniquely identified by issue_id, comment_id and type + // issue comment reaction is uniquely identified by issue_id, comment_id and type exists, err := sess.Exist(&issues_model.Reaction{ IssueID: reaction.IssueID, CommentID: reaction.CommentID, From 197489b5b15f5d0c7f0e6c35268c99d354f1dd6d Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sat, 16 Sep 2023 02:33:03 -0400 Subject: [PATCH 86/94] Fix lint --- services/repository/adopt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/repository/adopt.go b/services/repository/adopt.go index 00dce7295edb4..2f0e76e204b55 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -195,7 +195,7 @@ func adoptRepository(ctx context.Context, repoPath string, u *user_model.User, r return fmt.Errorf("updateRepository: %w", err) } - if err = repo_module.SyncReleasesWithTags(repo, gitRepo); err != nil { + if err = repo_module.SyncReleasesWithTags(repo, gitRepo, true); err != nil { return fmt.Errorf("SyncReleasesWithTags: %w", err) } From 049faf885098a62a799106499ff0f19a5753cb4c Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sat, 16 Sep 2023 02:44:49 -0400 Subject: [PATCH 87/94] Somehow git merge didn't delete this file --- modules/notification/notification.go | 380 --------------------------- 1 file changed, 380 deletions(-) delete mode 100644 modules/notification/notification.go diff --git a/modules/notification/notification.go b/modules/notification/notification.go deleted file mode 100644 index 7d890ecc78be2..0000000000000 --- a/modules/notification/notification.go +++ /dev/null @@ -1,380 +0,0 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package notification - -import ( - "context" - - issues_model "code.gitea.io/gitea/models/issues" - packages_model "code.gitea.io/gitea/models/packages" - repo_model "code.gitea.io/gitea/models/repo" - user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/notification/action" - "code.gitea.io/gitea/modules/notification/base" - "code.gitea.io/gitea/modules/notification/indexer" - "code.gitea.io/gitea/modules/notification/mail" - "code.gitea.io/gitea/modules/notification/mirror" - "code.gitea.io/gitea/modules/notification/ui" - "code.gitea.io/gitea/modules/repository" - "code.gitea.io/gitea/modules/setting" -) - -var notifiers []base.Notifier - -// RegisterNotifier providers method to receive notify messages -func RegisterNotifier(notifier base.Notifier) { - go notifier.Run() - notifiers = append(notifiers, notifier) -} - -// NewContext registers notification handlers -func NewContext() { - RegisterNotifier(ui.NewNotifier()) - if setting.Service.EnableNotifyMail { - RegisterNotifier(mail.NewNotifier()) - } - RegisterNotifier(indexer.NewNotifier()) - RegisterNotifier(action.NewNotifier()) - RegisterNotifier(mirror.NewNotifier()) -} - -// NotifyNewWikiPage notifies creating new wiki pages to notifiers -func NotifyNewWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, page, comment string) { - for _, notifier := range notifiers { - notifier.NotifyNewWikiPage(ctx, doer, repo, page, comment) - } -} - -// NotifyEditWikiPage notifies editing or renaming wiki pages to notifiers -func NotifyEditWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, page, comment string) { - for _, notifier := range notifiers { - notifier.NotifyEditWikiPage(ctx, doer, repo, page, comment) - } -} - -// NotifyDeleteWikiPage notifies deleting wiki pages to notifiers -func NotifyDeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, page string) { - for _, notifier := range notifiers { - notifier.NotifyDeleteWikiPage(ctx, doer, repo, page) - } -} - -// NotifyCreateIssueComment notifies issue comment related message to notifiers -func NotifyCreateIssueComment(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, - issue *issues_model.Issue, comment *issues_model.Comment, mentions []*user_model.User, -) { - for _, notifier := range notifiers { - notifier.NotifyCreateIssueComment(ctx, doer, repo, issue, comment, mentions) - } -} - -// NotifyNewIssue notifies new issue to notifiers -func NotifyNewIssue(ctx context.Context, issue *issues_model.Issue, mentions []*user_model.User) { - for _, notifier := range notifiers { - notifier.NotifyNewIssue(ctx, issue, mentions) - } -} - -// NotifyIssueChangeStatus notifies close or reopen issue to notifiers -func NotifyIssueChangeStatus(ctx context.Context, doer *user_model.User, commitID string, issue *issues_model.Issue, actionComment *issues_model.Comment, closeOrReopen bool) { - for _, notifier := range notifiers { - notifier.NotifyIssueChangeStatus(ctx, doer, commitID, issue, actionComment, closeOrReopen) - } -} - -// NotifyDeleteIssue notify when some issue deleted -func NotifyDeleteIssue(ctx context.Context, doer *user_model.User, issue *issues_model.Issue) { - for _, notifier := range notifiers { - notifier.NotifyDeleteIssue(ctx, doer, issue) - } -} - -// NotifyMergePullRequest notifies merge pull request to notifiers -func NotifyMergePullRequest(ctx context.Context, doer *user_model.User, pr *issues_model.PullRequest) { - for _, notifier := range notifiers { - notifier.NotifyMergePullRequest(ctx, doer, pr) - } -} - -// NotifyAutoMergePullRequest notifies merge pull request to notifiers -func NotifyAutoMergePullRequest(ctx context.Context, doer *user_model.User, pr *issues_model.PullRequest) { - for _, notifier := range notifiers { - notifier.NotifyAutoMergePullRequest(ctx, doer, pr) - } -} - -// NotifyNewPullRequest notifies new pull request to notifiers -func NotifyNewPullRequest(ctx context.Context, pr *issues_model.PullRequest, mentions []*user_model.User) { - if err := pr.LoadIssue(ctx); err != nil { - log.Error("%v", err) - return - } - if err := pr.Issue.LoadPoster(ctx); err != nil { - return - } - for _, notifier := range notifiers { - notifier.NotifyNewPullRequest(ctx, pr, mentions) - } -} - -// NotifyPullRequestSynchronized notifies Synchronized pull request -func NotifyPullRequestSynchronized(ctx context.Context, doer *user_model.User, pr *issues_model.PullRequest) { - for _, notifier := range notifiers { - notifier.NotifyPullRequestSynchronized(ctx, doer, pr) - } -} - -// NotifyPullRequestReview notifies new pull request review -func NotifyPullRequestReview(ctx context.Context, pr *issues_model.PullRequest, review *issues_model.Review, comment *issues_model.Comment, mentions []*user_model.User) { - if err := review.LoadReviewer(ctx); err != nil { - log.Error("%v", err) - return - } - for _, notifier := range notifiers { - notifier.NotifyPullRequestReview(ctx, pr, review, comment, mentions) - } -} - -// NotifyPullRequestCodeComment notifies new pull request code comment -func NotifyPullRequestCodeComment(ctx context.Context, pr *issues_model.PullRequest, comment *issues_model.Comment, mentions []*user_model.User) { - if err := comment.LoadPoster(ctx); err != nil { - log.Error("LoadPoster: %v", err) - return - } - for _, notifier := range notifiers { - notifier.NotifyPullRequestCodeComment(ctx, pr, comment, mentions) - } -} - -// NotifyPullRequestChangeTargetBranch notifies when a pull request's target branch was changed -func NotifyPullRequestChangeTargetBranch(ctx context.Context, doer *user_model.User, pr *issues_model.PullRequest, oldBranch string) { - for _, notifier := range notifiers { - notifier.NotifyPullRequestChangeTargetBranch(ctx, doer, pr, oldBranch) - } -} - -// NotifyPullRequestPushCommits notifies when push commits to pull request's head branch -func NotifyPullRequestPushCommits(ctx context.Context, doer *user_model.User, pr *issues_model.PullRequest, comment *issues_model.Comment) { - for _, notifier := range notifiers { - notifier.NotifyPullRequestPushCommits(ctx, doer, pr, comment) - } -} - -// NotifyPullReviewDismiss notifies when a review was dismissed by repo admin -func NotifyPullReviewDismiss(ctx context.Context, doer *user_model.User, review *issues_model.Review, comment *issues_model.Comment) { - for _, notifier := range notifiers { - notifier.NotifyPullReviewDismiss(ctx, doer, review, comment) - } -} - -// NotifyUpdateComment notifies update comment to notifiers -func NotifyUpdateComment(ctx context.Context, doer *user_model.User, c *issues_model.Comment, oldContent string) { - for _, notifier := range notifiers { - notifier.NotifyUpdateComment(ctx, doer, c, oldContent) - } -} - -// NotifyDeleteComment notifies delete comment to notifiers -func NotifyDeleteComment(ctx context.Context, doer *user_model.User, c *issues_model.Comment) { - for _, notifier := range notifiers { - notifier.NotifyDeleteComment(ctx, doer, c) - } -} - -// NotifyNewRelease notifies new release to notifiers -func NotifyNewRelease(ctx context.Context, rel *repo_model.Release) { - if err := rel.LoadAttributes(ctx); err != nil { - log.Error("LoadPublisher: %v", err) - return - } - for _, notifier := range notifiers { - notifier.NotifyNewRelease(ctx, rel) - } -} - -// NotifyUpdateRelease notifies update release to notifiers -func NotifyUpdateRelease(ctx context.Context, doer *user_model.User, rel *repo_model.Release) { - for _, notifier := range notifiers { - notifier.NotifyUpdateRelease(ctx, doer, rel) - } -} - -// NotifyDeleteRelease notifies delete release to notifiers -func NotifyDeleteRelease(ctx context.Context, doer *user_model.User, rel *repo_model.Release) { - for _, notifier := range notifiers { - notifier.NotifyDeleteRelease(ctx, doer, rel) - } -} - -// NotifyIssueChangeMilestone notifies change milestone to notifiers -func NotifyIssueChangeMilestone(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, oldMilestoneID int64) { - for _, notifier := range notifiers { - notifier.NotifyIssueChangeMilestone(ctx, doer, issue, oldMilestoneID) - } -} - -// NotifyIssueChangeContent notifies change content to notifiers -func NotifyIssueChangeContent(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, oldContent string) { - for _, notifier := range notifiers { - notifier.NotifyIssueChangeContent(ctx, doer, issue, oldContent) - } -} - -// NotifyIssueChangeAssignee notifies change content to notifiers -func NotifyIssueChangeAssignee(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, assignee *user_model.User, removed bool, comment *issues_model.Comment) { - for _, notifier := range notifiers { - notifier.NotifyIssueChangeAssignee(ctx, doer, issue, assignee, removed, comment) - } -} - -// NotifyPullRequestReviewRequest notifies Request Review change -func NotifyPullRequestReviewRequest(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, reviewer *user_model.User, isRequest bool, comment *issues_model.Comment) { - for _, notifier := range notifiers { - notifier.NotifyPullRequestReviewRequest(ctx, doer, issue, reviewer, isRequest, comment) - } -} - -// NotifyIssueClearLabels notifies clear labels to notifiers -func NotifyIssueClearLabels(ctx context.Context, doer *user_model.User, issue *issues_model.Issue) { - for _, notifier := range notifiers { - notifier.NotifyIssueClearLabels(ctx, doer, issue) - } -} - -// NotifyIssueChangeTitle notifies change title to notifiers -func NotifyIssueChangeTitle(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, oldTitle string) { - for _, notifier := range notifiers { - notifier.NotifyIssueChangeTitle(ctx, doer, issue, oldTitle) - } -} - -// NotifyIssueChangeRef notifies change reference to notifiers -func NotifyIssueChangeRef(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, oldRef string) { - for _, notifier := range notifiers { - notifier.NotifyIssueChangeRef(ctx, doer, issue, oldRef) - } -} - -// NotifyIssueChangeLabels notifies change labels to notifiers -func NotifyIssueChangeLabels(ctx context.Context, doer *user_model.User, issue *issues_model.Issue, - addedLabels, removedLabels []*issues_model.Label, -) { - for _, notifier := range notifiers { - notifier.NotifyIssueChangeLabels(ctx, doer, issue, addedLabels, removedLabels) - } -} - -// NotifyCreateRepository notifies create repository to notifiers -func NotifyCreateRepository(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository) { - for _, notifier := range notifiers { - notifier.NotifyCreateRepository(ctx, doer, u, repo) - } -} - -// NotifyAdoptRepository notifies the adoption of a repository to notifiers -func NotifyAdoptRepository(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository) { - for _, notifier := range notifiers { - notifier.NotifyAdoptRepository(ctx, doer, u, repo) - } -} - -// NotifyMigrateRepository notifies migrate repository to notifiers -func NotifyMigrateRepository(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository) { - for _, notifier := range notifiers { - notifier.NotifyMigrateRepository(ctx, doer, u, repo) - } -} - -// NotifyTransferRepository notifies create repository to notifiers -func NotifyTransferRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, newOwnerName string) { - for _, notifier := range notifiers { - notifier.NotifyTransferRepository(ctx, doer, repo, newOwnerName) - } -} - -// NotifyDeleteRepository notifies delete repository to notifiers -func NotifyDeleteRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository) { - for _, notifier := range notifiers { - notifier.NotifyDeleteRepository(ctx, doer, repo) - } -} - -// NotifyForkRepository notifies fork repository to notifiers -func NotifyForkRepository(ctx context.Context, doer *user_model.User, oldRepo, repo *repo_model.Repository) { - for _, notifier := range notifiers { - notifier.NotifyForkRepository(ctx, doer, oldRepo, repo) - } -} - -// NotifyRenameRepository notifies repository renamed -func NotifyRenameRepository(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, oldName string) { - for _, notifier := range notifiers { - notifier.NotifyRenameRepository(ctx, doer, repo, oldName) - } -} - -// NotifyPushCommits notifies commits pushed to notifiers -func NotifyPushCommits(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { - for _, notifier := range notifiers { - notifier.NotifyPushCommits(ctx, pusher, repo, opts, commits) - } -} - -// NotifyCreateRef notifies branch or tag creation to notifiers -func NotifyCreateRef(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, refFullName git.RefName, refID string) { - for _, notifier := range notifiers { - notifier.NotifyCreateRef(ctx, pusher, repo, refFullName, refID) - } -} - -// NotifyDeleteRef notifies branch or tag deletion to notifiers -func NotifyDeleteRef(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, refFullName git.RefName) { - for _, notifier := range notifiers { - notifier.NotifyDeleteRef(ctx, pusher, repo, refFullName) - } -} - -// NotifySyncPushCommits notifies commits pushed to notifiers -func NotifySyncPushCommits(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, opts *repository.PushUpdateOptions, commits *repository.PushCommits) { - for _, notifier := range notifiers { - notifier.NotifySyncPushCommits(ctx, pusher, repo, opts, commits) - } -} - -// NotifySyncCreateRef notifies branch or tag creation to notifiers -func NotifySyncCreateRef(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, refFullName git.RefName, refID string) { - for _, notifier := range notifiers { - notifier.NotifySyncCreateRef(ctx, pusher, repo, refFullName, refID) - } -} - -// NotifySyncDeleteRef notifies branch or tag deletion to notifiers -func NotifySyncDeleteRef(ctx context.Context, pusher *user_model.User, repo *repo_model.Repository, refFullName git.RefName) { - for _, notifier := range notifiers { - notifier.NotifySyncDeleteRef(ctx, pusher, repo, refFullName) - } -} - -// NotifyRepoPendingTransfer notifies creation of pending transfer to notifiers -func NotifyRepoPendingTransfer(ctx context.Context, doer, newOwner *user_model.User, repo *repo_model.Repository) { - for _, notifier := range notifiers { - notifier.NotifyRepoPendingTransfer(ctx, doer, newOwner, repo) - } -} - -// NotifyPackageCreate notifies creation of a package to notifiers -func NotifyPackageCreate(ctx context.Context, doer *user_model.User, pd *packages_model.PackageDescriptor) { - for _, notifier := range notifiers { - notifier.NotifyPackageCreate(ctx, doer, pd) - } -} - -// NotifyPackageDelete notifies deletion of a package to notifiers -func NotifyPackageDelete(ctx context.Context, doer *user_model.User, pd *packages_model.PackageDescriptor) { - for _, notifier := range notifiers { - notifier.NotifyPackageDelete(ctx, doer, pd) - } -} From 065469cffc527532208ec5e61a461a2845dcc8a4 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 29 Oct 2023 17:18:22 -0400 Subject: [PATCH 88/94] Fix functions that gets moved in #26969 --- models/issues/comment.go | 74 ++++ models/issues/issue.go | 83 ++++ models/issues/milestone.go | 82 ++++ models/issues/pull.go | 29 ++ models/migrate.go | 530 -------------------------- models/repo/release.go | 72 ++++ modules/context/repo.go | 2 +- services/migrations/gitea_uploader.go | 10 +- 8 files changed, 346 insertions(+), 536 deletions(-) delete mode 100644 models/migrate.go diff --git a/models/issues/comment.go b/models/issues/comment.go index 3928596c2e695..5080669943b25 100644 --- a/models/issues/comment.go +++ b/models/issues/comment.go @@ -1290,3 +1290,77 @@ func InsertIssueComments(ctx context.Context, comments []*Comment) error { } return committer.Commit() } + +// UpsertIssueComments inserts many comments of issues. +func UpsertIssueComments(ctx context.Context, comments []*Comment) error { + if len(comments) == 0 { + return nil + } + + issueIDs := make(map[int64]bool) + for _, comment := range comments { + issueIDs[comment.IssueID] = true + } + + return db.WithTx(ctx, func(ctx context.Context) error { + sess := db.GetEngine(ctx) + for _, comment := range comments { + exists, err := sess.Exist(&Comment{ + IssueID: comment.IssueID, + OriginalID: comment.OriginalID, + }) + if err != nil { + return err + } + if !exists { + if _, err := sess.NoAutoTime().Insert(comment); err != nil { + return err + } + } else { + if _, err := sess.NoAutoTime().Where( + "issue_id = ? AND original_id = ?", comment.IssueID, comment.OriginalID, + ).AllCols().Update(comment); err != nil { + return err + } + } + + for _, reaction := range comment.Reactions { + reaction.IssueID = comment.IssueID + reaction.CommentID = comment.ID + } + if len(comment.Reactions) > 0 { + for _, reaction := range comment.Reactions { + // issue comment reaction is uniquely identified by issue_id, comment_id and type + exists, err := sess.Exist(&Reaction{ + IssueID: reaction.IssueID, + CommentID: reaction.CommentID, + Type: reaction.Type, + }) + if err != nil { + return err + } + if exists { + if _, err := sess.Where( + "issue_id = ? AND comment_id = ? AND type = ?", + reaction.IssueID, reaction.CommentID, reaction.Type, + ).AllCols().Update(&reaction); err != nil { + return err + } + } else { + if _, err := sess.Insert(&reaction); err != nil { + return err + } + } + } + } + } + + for issueID := range issueIDs { + if _, err := db.Exec(ctx, "UPDATE issue SET num_comments = (SELECT count(*) FROM comment WHERE issue_id = ? AND `type`=?) WHERE id = ?", + issueID, CommentTypeComment, issueID); err != nil { + return err + } + } + return nil + }) +} diff --git a/models/issues/issue.go b/models/issues/issue.go index 8655b9de3fe42..0997d6e4b777c 100644 --- a/models/issues/issue.go +++ b/models/issues/issue.go @@ -939,3 +939,86 @@ func insertIssue(ctx context.Context, issue *Issue) error { return nil } + +// UpsertIssues creates new issues and updates existing issues in database +func UpsertIssues(ctx context.Context, issues ...*Issue) error { + return db.WithTx(ctx, func(ctx context.Context) error { + for _, issue := range issues { + if _, err := upsertIssue(ctx, issue); err != nil { + return err + } + } + return nil + }) +} + +func upsertIssue(ctx context.Context, issue *Issue) (isInsert bool, err error) { + sess := db.GetEngine(ctx) + + var issueIDs []int64 + err = sess.Table("issue").Where("repo_id = ? AND `index` = ?", issue.RepoID, issue.Index).Cols("id").Find(&issueIDs) + if err != nil { + return false, err + } + + if len(issueIDs) == 0 { + return true, insertIssue(ctx, issue) + } + + issue.ID = issueIDs[0] + return false, updateIssue(ctx, issue) +} + +func updateIssue(ctx context.Context, issue *Issue) error { + sess := db.GetEngine(ctx) + if _, err := sess.NoAutoTime().ID(issue.ID).AllCols().Update(issue); err != nil { + return err + } + issueLabels := resolveIssueLabels(issue.ID, issue.Labels) + if len(issueLabels) > 0 { + // delete old labels + if _, err := sess.Table("issue_label").Where("issue_id = ?", issue.ID).Delete(); err != nil { + return err + } + // insert new labels + if _, err := sess.Insert(issueLabels); err != nil { + return err + } + } + + for _, reaction := range issue.Reactions { + reaction.IssueID = issue.ID + } + + if len(issue.Reactions) > 0 { + // update existing reactions and insert new ones + for _, reaction := range issue.Reactions { + exists, err := sess.Exist(&Reaction{ID: reaction.ID}) + if err != nil { + return err + } + if exists { + if _, err := sess.ID(reaction.ID).AllCols().Update(&reaction); err != nil { + return err + } + } else { + if _, err := sess.Insert(&reaction); err != nil { + return err + } + } + } + } + + return nil +} + +func resolveIssueLabels(issueID int64, labels []*Label) []IssueLabel { + issueLabels := make([]IssueLabel, 0, len(labels)) + for _, label := range labels { + issueLabels = append(issueLabels, IssueLabel{ + IssueID: issueID, + LabelID: label.ID, + }) + } + return issueLabels +} diff --git a/models/issues/milestone.go b/models/issues/milestone.go index 66500c004d725..3c833fa2f56bd 100644 --- a/models/issues/milestone.go +++ b/models/issues/milestone.go @@ -383,3 +383,85 @@ func InsertMilestones(ctx context.Context, ms ...*Milestone) (err error) { } return committer.Commit() } + +// UpdateMilestones updates milestones of repository. +func UpdateMilestones(ctx context.Context, ms ...*Milestone) (err error) { + if len(ms) == 0 { + return nil + } + + return db.WithTx(ctx, func(ctx context.Context) error { + sess := db.GetEngine(ctx) + + // get existing milestones + existingMilestones := make([]*Milestone, 0) + if err = sess.Where("repo_id = ?", ms[0].RepoID).Find(&existingMilestones); err != nil { + return err + } + + milestonesToAdd := make([]*Milestone, 0) + milestonesToUpdate := make([]*Milestone, 0) + milestonesToDelete := make([]*Milestone, 0) + foundMap := make(map[int64]bool) + + openCount := 0 + closedCount := 0 + + for _, m := range ms { + var foundMilestone *Milestone + for _, existingMilestone := range existingMilestones { + if existingMilestone.OriginalID == m.OriginalID { + foundMilestone = existingMilestone + foundMap[existingMilestone.ID] = true + break + } + } + + if foundMilestone == nil { + milestonesToAdd = append(milestonesToAdd, m) + } else if foundMilestone.OriginalID != m.OriginalID { + m.ID = foundMilestone.ID + milestonesToUpdate = append(milestonesToUpdate, m) + } + + if m.IsClosed { + closedCount++ + } else { + openCount++ + } + } + + for _, existingMilestone := range existingMilestones { + if _, exist := foundMap[existingMilestone.ID]; !exist { + milestonesToDelete = append(milestonesToDelete, existingMilestone) + } + } + + if len(milestonesToAdd) > 0 { + if _, err = sess.Insert(milestonesToAdd); err != nil { + return err + } + } + + for _, m := range milestonesToUpdate { + if _, err = sess.ID(m.ID).AllCols().Update(m); err != nil { + return err + } + } + + for _, m := range milestonesToDelete { + if _, err = sess.ID(m.ID).Delete(m); err != nil { + return err + } + } + + if _, err = sess.ID(ms[0].RepoID).Update(&repo_model.Repository{ + NumMilestones: len(ms), + NumOpenMilestones: openCount, + NumClosedMilestones: closedCount, + }); err != nil { + return err + } + return nil + }) +} diff --git a/models/issues/pull.go b/models/issues/pull.go index 2379c61976635..9c1ea5d17a46d 100644 --- a/models/issues/pull.go +++ b/models/issues/pull.go @@ -1125,3 +1125,32 @@ func InsertPullRequests(ctx context.Context, prs ...*PullRequest) error { } return committer.Commit() } + +// UpsertPullRequests inserts new pull requests and updates existing pull requests in database +func UpsertPullRequests(ctx context.Context, prs ...*PullRequest) error { + if len(prs) == 0 { + return nil + } + + return db.WithTx(ctx, func(ctx context.Context) error { + sess := db.GetEngine(ctx) + for _, pr := range prs { + isInsert, err := upsertIssue(ctx, pr.Issue) + if err != nil { + return err + } + pr.IssueID = pr.Issue.ID + + if isInsert { + if _, err := sess.NoAutoTime().Insert(pr); err != nil { + return err + } + } else { + if _, err := sess.NoAutoTime().ID(pr.ID).AllCols().Update(pr); err != nil { + return err + } + } + } + return nil + }) +} diff --git a/models/migrate.go b/models/migrate.go deleted file mode 100644 index f38b35eb82a86..0000000000000 --- a/models/migrate.go +++ /dev/null @@ -1,530 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package models - -import ( - "context" - - "code.gitea.io/gitea/models/db" - issues_model "code.gitea.io/gitea/models/issues" - repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/modules/container" - "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/structs" -) - -// InsertMilestones creates milestones of repository. -func InsertMilestones(ms ...*issues_model.Milestone) (err error) { - if len(ms) == 0 { - return nil - } - - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - sess := db.GetEngine(ctx) - - // to return the id, so we should not use batch insert - for _, m := range ms { - if _, err = sess.NoAutoTime().Insert(m); err != nil { - return err - } - } - - if _, err = sess.Exec("UPDATE `repository` SET num_milestones = num_milestones + ? WHERE id = ?", len(ms), ms[0].RepoID); err != nil { - return err - } - return committer.Commit() -} - -// UpdateMilestones updates milestones of repository. -func UpdateMilestones(ctx context.Context, ms ...*issues_model.Milestone) (err error) { - if len(ms) == 0 { - return nil - } - - return db.WithTx(ctx, func(ctx context.Context) error { - sess := db.GetEngine(ctx) - - // get existing milestones - existingMilestones := make([]*issues_model.Milestone, 0) - if err = sess.Where("repo_id = ?", ms[0].RepoID).Find(&existingMilestones); err != nil { - return err - } - - milestonesToAdd := make([]*issues_model.Milestone, 0) - milestonesToUpdate := make([]*issues_model.Milestone, 0) - milestonesToDelete := make([]*issues_model.Milestone, 0) - foundMap := make(map[int64]bool) - - openCount := 0 - closedCount := 0 - - for _, m := range ms { - var foundMilestone *issues_model.Milestone - for _, existingMilestone := range existingMilestones { - if existingMilestone.OriginalID == m.OriginalID { - foundMilestone = existingMilestone - foundMap[existingMilestone.ID] = true - break - } - } - - if foundMilestone == nil { - milestonesToAdd = append(milestonesToAdd, m) - } else if foundMilestone.OriginalID != m.OriginalID { - m.ID = foundMilestone.ID - milestonesToUpdate = append(milestonesToUpdate, m) - } - - if m.IsClosed { - closedCount++ - } else { - openCount++ - } - } - - for _, existingMilestone := range existingMilestones { - if _, exist := foundMap[existingMilestone.ID]; !exist { - milestonesToDelete = append(milestonesToDelete, existingMilestone) - } - } - - if len(milestonesToAdd) > 0 { - if _, err = sess.Insert(milestonesToAdd); err != nil { - return err - } - } - - for _, m := range milestonesToUpdate { - if _, err = sess.ID(m.ID).AllCols().Update(m); err != nil { - return err - } - } - - for _, m := range milestonesToDelete { - if _, err = sess.ID(m.ID).Delete(m); err != nil { - return err - } - } - - if _, err = sess.ID(ms[0].RepoID).Update(&repo_model.Repository{ - NumMilestones: len(ms), - NumOpenMilestones: openCount, - NumClosedMilestones: closedCount, - }); err != nil { - return err - } - return nil - }) -} - -// InsertIssues insert issues to database -func InsertIssues(issues ...*issues_model.Issue) error { - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - - for _, issue := range issues { - if err := insertIssue(ctx, issue); err != nil { - return err - } - } - return committer.Commit() -} - -func resolveIssueLabels(issueID int64, labels []*issues_model.Label) []issues_model.IssueLabel { - issueLabels := make([]issues_model.IssueLabel, 0, len(labels)) - for _, label := range labels { - issueLabels = append(issueLabels, issues_model.IssueLabel{ - IssueID: issueID, - LabelID: label.ID, - }) - } - return issueLabels -} - -func insertIssue(ctx context.Context, issue *issues_model.Issue) error { - sess := db.GetEngine(ctx) - if _, err := sess.NoAutoTime().Insert(issue); err != nil { - return err - } - issueLabels := resolveIssueLabels(issue.ID, issue.Labels) - if len(issueLabels) > 0 { - if _, err := sess.Insert(issueLabels); err != nil { - return err - } - } - - for _, reaction := range issue.Reactions { - reaction.IssueID = issue.ID - } - - if len(issue.Reactions) > 0 { - if _, err := sess.Insert(issue.Reactions); err != nil { - return err - } - } - - return nil -} - -// UpsertIssues creates new issues and updates existing issues in database -func UpsertIssues(ctx context.Context, issues ...*issues_model.Issue) error { - return db.WithTx(ctx, func(ctx context.Context) error { - for _, issue := range issues { - if _, err := upsertIssue(ctx, issue); err != nil { - return err - } - } - return nil - }) -} - -func updateIssue(ctx context.Context, issue *issues_model.Issue) error { - sess := db.GetEngine(ctx) - if _, err := sess.NoAutoTime().ID(issue.ID).AllCols().Update(issue); err != nil { - return err - } - issueLabels := resolveIssueLabels(issue.ID, issue.Labels) - if len(issueLabels) > 0 { - // delete old labels - if _, err := sess.Table("issue_label").Where("issue_id = ?", issue.ID).Delete(); err != nil { - return err - } - // insert new labels - if _, err := sess.Insert(issueLabels); err != nil { - return err - } - } - - for _, reaction := range issue.Reactions { - reaction.IssueID = issue.ID - } - - if len(issue.Reactions) > 0 { - // update existing reactions and insert new ones - for _, reaction := range issue.Reactions { - exists, err := sess.Exist(&issues_model.Reaction{ID: reaction.ID}) - if err != nil { - return err - } - if exists { - if _, err := sess.ID(reaction.ID).AllCols().Update(&reaction); err != nil { - return err - } - } else { - if _, err := sess.Insert(&reaction); err != nil { - return err - } - } - } - } - - return nil -} - -func upsertIssue(ctx context.Context, issue *issues_model.Issue) (isInsert bool, err error) { - sess := db.GetEngine(ctx) - - var issueIDs []int64 - err = sess.Table("issue").Where("repo_id = ? AND `index` = ?", issue.RepoID, issue.Index).Cols("id").Find(&issueIDs) - if err != nil { - return false, err - } - - if len(issueIDs) == 0 { - return true, insertIssue(ctx, issue) - } - - issue.ID = issueIDs[0] - return false, updateIssue(ctx, issue) -} - -// InsertIssueComments inserts many comments of issues. -func InsertIssueComments(comments []*issues_model.Comment) error { - if len(comments) == 0 { - return nil - } - - issueIDs := make(container.Set[int64]) - for _, comment := range comments { - issueIDs.Add(comment.IssueID) - } - - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - for _, comment := range comments { - if _, err := db.GetEngine(ctx).NoAutoTime().Insert(comment); err != nil { - return err - } - - for _, reaction := range comment.Reactions { - reaction.IssueID = comment.IssueID - reaction.CommentID = comment.ID - } - if len(comment.Reactions) > 0 { - if err := db.Insert(ctx, comment.Reactions); err != nil { - return err - } - } - } - - for issueID := range issueIDs { - if _, err := db.Exec(ctx, "UPDATE issue set num_comments = (SELECT count(*) FROM comment WHERE issue_id = ? AND `type`=?) WHERE id = ?", - issueID, issues_model.CommentTypeComment, issueID); err != nil { - return err - } - } - return committer.Commit() -} - -// UpsertIssueComments inserts many comments of issues. -func UpsertIssueComments(ctx context.Context, comments []*issues_model.Comment) error { - if len(comments) == 0 { - return nil - } - - issueIDs := make(map[int64]bool) - for _, comment := range comments { - issueIDs[comment.IssueID] = true - } - - return db.WithTx(ctx, func(ctx context.Context) error { - sess := db.GetEngine(ctx) - for _, comment := range comments { - exists, err := sess.Exist(&issues_model.Comment{ - IssueID: comment.IssueID, - OriginalID: comment.OriginalID, - }) - if err != nil { - return err - } - if !exists { - if _, err := sess.NoAutoTime().Insert(comment); err != nil { - return err - } - } else { - if _, err := sess.NoAutoTime().Where( - "issue_id = ? AND original_id = ?", comment.IssueID, comment.OriginalID, - ).AllCols().Update(comment); err != nil { - return err - } - } - - for _, reaction := range comment.Reactions { - reaction.IssueID = comment.IssueID - reaction.CommentID = comment.ID - } - if len(comment.Reactions) > 0 { - for _, reaction := range comment.Reactions { - // issue comment reaction is uniquely identified by issue_id, comment_id and type - exists, err := sess.Exist(&issues_model.Reaction{ - IssueID: reaction.IssueID, - CommentID: reaction.CommentID, - Type: reaction.Type, - }) - if err != nil { - return err - } - if exists { - if _, err := sess.Where( - "issue_id = ? AND comment_id = ? AND type = ?", - reaction.IssueID, reaction.CommentID, reaction.Type, - ).AllCols().Update(&reaction); err != nil { - return err - } - } else { - if _, err := sess.Insert(&reaction); err != nil { - return err - } - } - } - } - } - - for issueID := range issueIDs { - if _, err := db.Exec(ctx, "UPDATE issue SET num_comments = (SELECT count(*) FROM comment WHERE issue_id = ? AND `type`=?) WHERE id = ?", - issueID, issues_model.CommentTypeComment, issueID); err != nil { - return err - } - } - return nil - }) -} - -// InsertPullRequests inserted pull requests -func InsertPullRequests(ctx context.Context, prs ...*issues_model.PullRequest) error { - ctx, committer, err := db.TxContext(ctx) - if err != nil { - return err - } - defer committer.Close() - sess := db.GetEngine(ctx) - for _, pr := range prs { - if err := insertIssue(ctx, pr.Issue); err != nil { - return err - } - pr.IssueID = pr.Issue.ID - if _, err := sess.NoAutoTime().Insert(pr); err != nil { - return err - } - } - return committer.Commit() -} - -// UpsertPullRequests inserts new pull requests and updates existing pull requests in database -func UpsertPullRequests(ctx context.Context, prs ...*issues_model.PullRequest) error { - if len(prs) == 0 { - return nil - } - - return db.WithTx(ctx, func(ctx context.Context) error { - sess := db.GetEngine(ctx) - for _, pr := range prs { - isInsert, err := upsertIssue(ctx, pr.Issue) - if err != nil { - return err - } - pr.IssueID = pr.Issue.ID - - if isInsert { - if _, err := sess.NoAutoTime().Insert(pr); err != nil { - return err - } - } else { - if _, err := sess.NoAutoTime().ID(pr.ID).AllCols().Update(pr); err != nil { - return err - } - } - } - return nil - }) -} - -// InsertReleases migrates release -func InsertReleases(rels ...*repo_model.Release) error { - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - sess := db.GetEngine(ctx) - - for _, rel := range rels { - if _, err := sess.NoAutoTime().Insert(rel); err != nil { - return err - } - - if len(rel.Attachments) > 0 { - for i := range rel.Attachments { - rel.Attachments[i].ReleaseID = rel.ID - } - - if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil { - return err - } - } - } - - return committer.Commit() -} - -// UpsertReleases inserts new releases and updates existing releases -func UpsertReleases(rels ...*repo_model.Release) error { - ctx, committer, err := db.TxContext(db.DefaultContext) - if err != nil { - return err - } - defer committer.Close() - sess := db.GetEngine(ctx) - - for _, rel := range rels { - exists, err := sess.Where("repo_id = ? AND tag_name = ?", rel.RepoID, rel.TagName).Exist(&repo_model.Release{}) - if err != nil { - return err - } - - if !exists { - if _, err := sess.NoAutoTime().Insert(rel); err != nil { - return err - } - - if len(rel.Attachments) > 0 { - for i := range rel.Attachments { - rel.Attachments[i].ReleaseID = rel.ID - } - - if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil { - return err - } - } - } else { - if _, err := sess.NoAutoTime(). - Where("repo_id = ? AND tag_name = ?", rel.RepoID, rel.TagName). - AllCols().Update(rel); err != nil { - return err - } - - if len(rel.Attachments) > 0 { - for i := range rel.Attachments { - rel.Attachments[i].ReleaseID = rel.ID - } - - var existingReleases []*repo_model.Attachment - err := sess.Where("release_id = ?", rel.ID).Find(&existingReleases) - if err != nil { - return err - } - - if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil { - return err - } - - var ids []int64 - for _, existingRelease := range existingReleases { - // TODO: file operations are not atomic, so errors should be handled - err = storage.Attachments.Delete(existingRelease.RelativePath()) - if err != nil { - return err - } - - ids = append(ids, existingRelease.ID) - } - if _, err := sess.NoAutoTime().In("id", ids).Delete(&repo_model.Attachment{}); err != nil { - return err - } - } - } - } - - return committer.Commit() -} - -// UpdateMigrationsByType updates all migrated repositories' posterid from gitServiceType to replace originalAuthorID to posterID -func UpdateMigrationsByType(tp structs.GitServiceType, externalUserID string, userID int64) error { - if err := issues_model.UpdateIssuesMigrationsByType(tp, externalUserID, userID); err != nil { - return err - } - - if err := issues_model.UpdateCommentsMigrationsByType(tp, externalUserID, userID); err != nil { - return err - } - - if err := repo_model.UpdateReleasesMigrationsByType(tp, externalUserID, userID); err != nil { - return err - } - - if err := issues_model.UpdateReactionsMigrationsByType(tp, externalUserID, userID); err != nil { - return err - } - return issues_model.UpdateReviewsMigrationsByType(tp, externalUserID, userID) -} diff --git a/models/repo/release.go b/models/repo/release.go index ff31ec451025e..a20ce50b66da8 100644 --- a/models/repo/release.go +++ b/models/repo/release.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" @@ -580,3 +581,74 @@ func InsertReleases(ctx context.Context, rels ...*Release) error { return committer.Commit() } + +// UpsertReleases inserts new releases and updates existing releases +func UpsertReleases(ctx context.Context, rels ...*Release) error { + ctx, committer, err := db.TxContext(ctx) + if err != nil { + return err + } + defer committer.Close() + sess := db.GetEngine(ctx) + + for _, rel := range rels { + exists, err := sess.Where("repo_id = ? AND tag_name = ?", rel.RepoID, rel.TagName).Exist(&Release{}) + if err != nil { + return err + } + + if !exists { + if _, err := sess.NoAutoTime().Insert(rel); err != nil { + return err + } + + if len(rel.Attachments) > 0 { + for i := range rel.Attachments { + rel.Attachments[i].ReleaseID = rel.ID + } + + if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil { + return err + } + } + } else { + if _, err := sess.NoAutoTime(). + Where("repo_id = ? AND tag_name = ?", rel.RepoID, rel.TagName). + AllCols().Update(rel); err != nil { + return err + } + + if len(rel.Attachments) > 0 { + for i := range rel.Attachments { + rel.Attachments[i].ReleaseID = rel.ID + } + + var existingReleases []*Attachment + err := sess.Where("release_id = ?", rel.ID).Find(&existingReleases) + if err != nil { + return err + } + + if _, err := sess.NoAutoTime().Insert(rel.Attachments); err != nil { + return err + } + + var ids []int64 + for _, existingRelease := range existingReleases { + // TODO: file operations are not atomic, so errors should be handled + err = storage.Attachments.Delete(existingRelease.RelativePath()) + if err != nil { + return err + } + + ids = append(ids, existingRelease.ID) + } + if _, err := sess.NoAutoTime().In("id", ids).Delete(&Attachment{}); err != nil { + return err + } + } + } + } + + return committer.Commit() +} diff --git a/modules/context/repo.go b/modules/context/repo.go index f28b86737d9b5..585daf7a940c9 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -728,7 +728,7 @@ func RepoAssignment(ctx *Context) context.CancelFunc { } ctx.Data["CanCompareOrPull"] = canCompare ctx.Data["PullRequestCtx"] = ctx.Repo.PullRequest - ctx.Data["AllowsIssues"] = repo.AllowsIssues() + ctx.Data["AllowsIssues"] = repo.AllowsIssues(ctx) if ctx.Repo.Repository.Status == repo_model.RepositoryPendingTransfer { repoTransfer, err := models.GetPendingRepositoryTransfer(ctx, ctx.Repo.Repository) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 1c0826ac42db0..7d757d922cf9a 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -996,7 +996,7 @@ func (g *GiteaLocalUploader) UpdateTopics(topics ...string) error { func (g *GiteaLocalUploader) UpdateMilestones(milestones ...*base.Milestone) error { mss := g.prepareMilestones(milestones...) - err := models.UpdateMilestones(g.ctx, mss...) + err := issues_model.UpdateMilestones(g.ctx, mss...) if err != nil { return err } @@ -1025,7 +1025,7 @@ func (g *GiteaLocalUploader) PatchReleases(releases ...*base.Release) error { return err } - return models.UpsertReleases(rels...) + return repo_model.UpsertReleases(g.ctx, rels...) } func (g *GiteaLocalUploader) PatchIssues(issues ...*base.Issue) error { @@ -1037,7 +1037,7 @@ func (g *GiteaLocalUploader) PatchIssues(issues ...*base.Issue) error { return nil } - if err := models.UpsertIssues(g.ctx, iss...); err != nil { + if err := issues_model.UpsertIssues(g.ctx, iss...); err != nil { return err } @@ -1056,7 +1056,7 @@ func (g *GiteaLocalUploader) PatchComments(comments ...*base.Comment) error { if len(cms) == 0 { return nil } - return models.UpsertIssueComments(g.ctx, cms) + return issues_model.UpsertIssueComments(g.ctx, cms) } func (g *GiteaLocalUploader) PatchPullRequests(prs ...*base.PullRequest) error { @@ -1064,7 +1064,7 @@ func (g *GiteaLocalUploader) PatchPullRequests(prs ...*base.PullRequest) error { if err != nil { return err } - if err := models.UpsertPullRequests(g.ctx, gprs...); err != nil { + if err := issues_model.UpsertPullRequests(g.ctx, gprs...); err != nil { return err } for _, pr := range gprs { From 218c8f535c289c586479a2401cd9e376ee5ffc9b Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Wed, 8 Nov 2023 00:24:42 -0500 Subject: [PATCH 89/94] Remove unnecessary variables --- cmd/admin.go | 3 +-- modules/repository/repo.go | 3 +-- services/migrations/gitea_uploader.go | 3 +-- services/mirror/mirror_pull.go | 3 +-- services/repository/adopt.go | 3 +-- services/repository/fork.go | 3 +-- 6 files changed, 6 insertions(+), 12 deletions(-) diff --git a/cmd/admin.go b/cmd/admin.go index f2753d5efdfbe..11b065294b32b 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -134,8 +134,7 @@ func runRepoSyncReleases(_ *cli.Context) error { } log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum) - tagOnlyReleases := false - if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo, tagOnlyReleases); err != nil { + if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo, false); err != nil { log.Warn(" SyncReleasesWithTags: %v", err) gitRepo.Close() continue diff --git a/modules/repository/repo.go b/modules/repository/repo.go index ea47a2dd3dd3d..5c45941c18076 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -159,8 +159,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, // note: this will greatly improve release (tag) sync // for pull-mirrors with many tags repo.IsMirror = opts.Mirror - tagOnlyReleases := true - if err = SyncReleasesWithTags(ctx, repo, gitRepo, tagOnlyReleases); err != nil { + if err = SyncReleasesWithTags(ctx, repo, gitRepo, true); err != nil { log.Error("Failed to synchronize tags to releases for repository: %v", err) } } diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 7d757d922cf9a..3c9dcd2a5a4f6 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -384,8 +384,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error { // SyncTags syncs releases with tags in the databases func (g *GiteaLocalUploader) SyncTags() error { - tagOnlyReleases := false - return repo_module.SyncReleasesWithTags(g.ctx, g.repo, g.gitRepo, tagOnlyReleases) + return repo_module.SyncReleasesWithTags(g.ctx, g.repo, g.gitRepo, false) } func (g *GiteaLocalUploader) prepareIssues(issues ...*base.Issue) ([]*issues_model.Issue, error) { diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index ab404cadf0002..f12806ad60aeb 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -314,8 +314,7 @@ func runSyncGit(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, } log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) - tagOnlyReleases := false - if err = repo_module.SyncReleasesWithTags(ctx, m.Repo, gitRepo, tagOnlyReleases); err != nil { + if err = repo_module.SyncReleasesWithTags(ctx, m.Repo, gitRepo, false); err != nil { log.Error("SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v", m.Repo, err) } diff --git a/services/repository/adopt.go b/services/repository/adopt.go index 5e55f5456f22d..1966ffda9d0ba 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -195,8 +195,7 @@ func adoptRepository(ctx context.Context, repoPath string, u *user_model.User, r return fmt.Errorf("updateRepository: %w", err) } - tagOnlyReleases := true - if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo, tagOnlyReleases); err != nil { + if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo, true); err != nil { return fmt.Errorf("SyncReleasesWithTags: %w", err) } diff --git a/services/repository/fork.go b/services/repository/fork.go index 43994d64f5bff..1c334655a3fb5 100644 --- a/services/repository/fork.go +++ b/services/repository/fork.go @@ -194,8 +194,7 @@ func ForkRepository(ctx context.Context, doer, owner *user_model.User, opts Fork log.Error("Open created git repository failed: %v", err) } else { defer gitRepo.Close() - tagOnlyReleases := false - if err := repo_module.SyncReleasesWithTags(ctx, repo, gitRepo, tagOnlyReleases); err != nil { + if err := repo_module.SyncReleasesWithTags(ctx, repo, gitRepo, false); err != nil { log.Error("Sync releases from git tags failed: %v", err) } } From 77fd657b0a35a4ce74249d09aea798f1ec355af3 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Tue, 6 Feb 2024 15:56:02 -0500 Subject: [PATCH 90/94] Fix build issues from merge --- modules/repository/repo.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index c2a3adc318971..db3bdda0db2e1 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -305,7 +305,7 @@ func SyncRepoTags(ctx context.Context, repoID int64) error { } defer gitRepo.Close() - return SyncReleasesWithTags(ctx, repo, gitRepo) + return SyncReleasesWithTags(ctx, repo, gitRepo, false) } // SyncReleasesWithTags synchronizes release table with repository tags for each of the releases. @@ -530,14 +530,14 @@ func (shortRelease) TableName() string { return "release" } -// pullMirrorReleaseSync is a pull-mirror specific tag<->release table +// recreateMirrorReleaseFromTags is a pull-mirror specific tag<->release table // synchronization which overwrites all Releases from the repository tags. This // can be relied on since a pull-mirror is always identical to its // upstream. Hence, after each sync we want the pull-mirror release set to be // identical to the upstream tag set. This is much more efficient for // repositories like https://github.com/vim/vim (with over 13000 tags). -func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { - log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) +func recreateMirrorReleaseFromTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { + log.Trace("recreateMirrorReleaseFromTags: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) tags, numTags, err := gitRepo.GetTagInfos(0, 0) if err != nil { return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) From 18fffc0a06f9983398a1813180740e3e283bb074 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 25 Feb 2024 19:29:53 -0500 Subject: [PATCH 91/94] Apply suggestion --- models/issues/issue.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/models/issues/issue.go b/models/issues/issue.go index 96bf470a1da40..4929d5ed92f5b 100644 --- a/models/issues/issue.go +++ b/models/issues/issue.go @@ -937,18 +937,13 @@ func UpsertIssues(ctx context.Context, issues ...*Issue) error { func upsertIssue(ctx context.Context, issue *Issue) (isInsert bool, err error) { sess := db.GetEngine(ctx) - - var issueIDs []int64 - err = sess.Table("issue").Where("repo_id = ? AND `index` = ?", issue.RepoID, issue.Index).Cols("id").Find(&issueIDs) + has, err := sess.Table("issue").Where("repo_id = ? AND `index` = ?", issue.RepoID, issue.Index).Cols("id").Get(&issue.ID) if err != nil { return false, err } - - if len(issueIDs) == 0 { + if !has { return true, insertIssue(ctx, issue) } - - issue.ID = issueIDs[0] return false, updateIssue(ctx, issue) } From e677d96348ba301c60137cd6920303a713be7a0b Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sun, 25 Feb 2024 19:38:47 -0500 Subject: [PATCH 92/94] Avoid jquery --- web_src/js/features/repo-migration.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/features/repo-migration.js b/web_src/js/features/repo-migration.js index 2a2978857cf09..309a90f70c0ec 100644 --- a/web_src/js/features/repo-migration.js +++ b/web_src/js/features/repo-migration.js @@ -51,7 +51,7 @@ function checkItems(tokenAuth) { } else { enableItems = user?.value !== '' || pass?.value !== ''; } - if (enableItems && Number(service?.value) > 1 && !allowedServiceTypes.includes(Number($service.val()))) { + if (enableItems && Number(service?.value) > 1 && !allowedServiceTypes.includes(Number(service?.value))) { if (mirror?.checked) { for (const item of items) { item.disabled = item.name !== 'wiki'; From e8119df983cb45a021aa86ff6c5dca97a68d492a Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sat, 2 Mar 2024 05:54:16 -0500 Subject: [PATCH 93/94] Fix build errors --- services/repository/migrate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/repository/migrate.go b/services/repository/migrate.go index 51fdd90f5400e..e93ac4348340b 100644 --- a/services/repository/migrate.go +++ b/services/repository/migrate.go @@ -138,7 +138,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, // note: this will greatly improve release (tag) sync // for pull-mirrors with many tags repo.IsMirror = opts.Mirror - if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil { + if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo, true); err != nil { log.Error("Failed to synchronize tags to releases for repository: %v", err) } } From d706aa54d0f849760625bc9fc54db54343d5283c Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sat, 27 Apr 2024 16:12:18 -0400 Subject: [PATCH 94/94] Change `interface{}` to `any` --- modules/migration/comment.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/migration/comment.go b/modules/migration/comment.go index bd1a8a9b66462..a46aa5989ae55 100644 --- a/modules/migration/comment.go +++ b/modules/migration/comment.go @@ -24,8 +24,8 @@ type Comment struct { Updated time.Time Content string Reactions []*Reaction - Meta map[string]interface{} `yaml:"meta,omitempty"` // see models/issues/comment.go for fields in Comment struct - OriginalID int64 `yaml:"-"` // ID from the upstream syncing source + Meta map[string]any `yaml:"meta,omitempty"` // see models/issues/comment.go for fields in Comment struct + OriginalID int64 `yaml:"-"` // ID from the upstream syncing source } // GetExternalName ExternalUserMigrated interface