Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sync issues, PRs, labels, milestones, comments from GitHub mirrors #20311

Open
wants to merge 136 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 96 commits
Commits
Show all changes
136 commits
Select commit Hold shift + click to select a range
6f51310
Define interfaces syncing new issues/PRs
harryzcy Jul 10, 2022
2bcb4c9
Support getting new issues from github
harryzcy Jul 10, 2022
c54aa4e
Put issue reactions code in a seperate function
harryzcy Jul 10, 2022
8532fa3
Rename function to getIssueReactions
harryzcy Jul 10, 2022
ad8eb2b
Support getting new PRs from github
harryzcy Jul 10, 2022
40c3b32
Fix issue that new PRs not returned
harryzcy Jul 10, 2022
039e505
Allow migration items with mirroring for GitHub
harryzcy Jul 11, 2022
a3ca5b0
Fix a comment
harryzcy Jul 11, 2022
176009e
Create relevant functions for uploader
harryzcy Jul 11, 2022
698d6be
Optimize topics creation and updates
harryzcy Jul 11, 2022
fb82294
Support updating labels
harryzcy Jul 12, 2022
22835e4
Fix error not returned error
harryzcy Jul 12, 2022
98a8e69
Update Uploader interface
harryzcy Jul 12, 2022
7b7b166
Support updating issues
harryzcy Jul 12, 2022
0932caa
Some cleanups
harryzcy Jul 12, 2022
5ffadeb
Support updating pull requests
harryzcy Jul 12, 2022
6d24299
Support updating milestones
harryzcy Jul 12, 2022
faa0d0b
Fix milestone migrations
harryzcy Jul 12, 2022
5895807
Fix: exec sql with session
harryzcy Jul 13, 2022
b96fe66
Fix test
harryzcy Jul 13, 2022
696f58f
Merge remote-tracking branch 'upstream/main' into sync-issue-pr-and-more
harryzcy Jul 13, 2022
8127c25
Provide preliminary implementation for releases
harryzcy Jul 14, 2022
07ba080
Support updating comments
harryzcy Jul 14, 2022
9892ae1
Add closed date to milestones
harryzcy Jul 15, 2022
97d47b2
Fix xorm usage
harryzcy Jul 15, 2022
9f5721f
Revert previous change
harryzcy Jul 15, 2022
9ac5518
Fix issue updates
harryzcy Jul 15, 2022
2ae6e4c
Add sync repository function
harryzcy Jul 16, 2022
06cc290
Get new comments/reviews
harryzcy Jul 16, 2022
aa7f59e
Merge remote-tracking branch 'upstream/main' into sync-issue-pr-and-more
harryzcy Jul 16, 2022
5b1c73e
Allow syncing with mirror option checked
harryzcy Jul 16, 2022
f627311
Display pull request for mirror repo
harryzcy Jul 16, 2022
9ee3ba5
Disable issue on mirror repo
harryzcy Jul 16, 2022
5a1dda4
Merge remote-tracking branch 'upstream/main' into sync-issue-pr-and-more
harryzcy Jul 19, 2022
4f67c5d
Merge remote-tracking branch 'upstream/main' into sync-issue-pr-and-more
harryzcy Jul 21, 2022
bbd4648
Add sync items to Mirror struct
harryzcy Jul 22, 2022
a0303a2
Support syncing reviews
harryzcy Jul 22, 2022
8874bf5
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Jul 22, 2022
dd20d7d
Merge remote-tracking branch 'upstream/main' into sync-issue-pr-and-more
harryzcy Aug 22, 2022
c733415
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Apr 6, 2023
73a5a7c
Fix lint issue caused by merge
harryzcy Apr 6, 2023
12851f2
Avoid duplicate code
harryzcy Apr 6, 2023
e3a16ff
Fix SPDX-License-Identifier
harryzcy Apr 6, 2023
e7b9e02
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Apr 6, 2023
1c8b8ef
Add comments
harryzcy Apr 6, 2023
9af6292
Fix error when syncing comments when PR is not synced
harryzcy Apr 6, 2023
acd0ad2
CheckAndEnsureSafePR was ignored when merging
harryzcy Apr 6, 2023
6b8300f
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Apr 8, 2023
9887d8d
Fix download pr tests
harryzcy Apr 8, 2023
8481bcf
Correctly format ISO 8601 time
harryzcy Apr 8, 2023
448ccfd
Update comments
harryzcy Apr 8, 2023
8fb9956
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Apr 8, 2023
786d15f
Fix a sql query
harryzcy Apr 9, 2023
a2df3f1
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Apr 22, 2023
d9cfdb1
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Apr 22, 2023
b5edfea
Merge branch 'main' into sync-issue-pr-and-more
harryzcy May 3, 2023
e50ab38
Fix lint
harryzcy May 4, 2023
d83b2ef
Merge branch 'main' into sync-issue-pr-and-more
harryzcy May 6, 2023
15d16a5
Identify the right place to do continuous sync
harryzcy May 6, 2023
ce51877
Sync mirror after migration
harryzcy May 6, 2023
78945eb
Enable github sync flag and fix panics
harryzcy May 6, 2023
a3a4015
Correctly store mirror setting in migration
harryzcy May 6, 2023
b8f8ab4
Fix: don't delete repo when sync failed
harryzcy May 7, 2023
de2f2a7
Fix error when syncing milestones
harryzcy May 7, 2023
3a72ed9
Update label identification method
harryzcy May 7, 2023
ef82e4e
Fix lint
harryzcy May 7, 2023
244c263
Merge branch 'main' into sync-issue-pr-and-more
harryzcy May 8, 2023
e52964c
Use correct last synced time
harryzcy May 15, 2023
82fe9d5
Correctly get issue id
harryzcy May 15, 2023
81cf9fc
Fix PR request sync issue
harryzcy May 16, 2023
994da85
Merge branch 'main' into sync-issue-pr-and-more
harryzcy May 16, 2023
6855c81
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Jun 24, 2023
6e3b980
Add OriginalID for milestones
harryzcy Jun 24, 2023
186a9a9
Add OriginalID to comments
harryzcy Jun 24, 2023
837c09e
Update comment for label struct
harryzcy Jun 24, 2023
b094125
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Jul 21, 2023
cbed118
Add OriginalID to Release
harryzcy Jul 21, 2023
2b31c33
Won't hit rate limit with github actions
harryzcy Jul 21, 2023
3d9b7ed
Fix: include originalID in getAllCommentsSince
harryzcy Jul 22, 2023
f7e9422
Identify comment by original_id only
harryzcy Jul 22, 2023
328fdaf
Add OriginalID to reviews and review comments
harryzcy Jul 22, 2023
78a00af
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Jul 22, 2023
87c9238
Enable TestGiteaUploadRepo only when token
harryzcy Jul 22, 2023
a48b1f3
Fix: update all cols including false boolean value
harryzcy Jul 22, 2023
0fedce2
Comment out code that cause bug with releases
harryzcy Jul 22, 2023
d8c0634
Another place to comment out
harryzcy Jul 22, 2023
7406559
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Jul 22, 2023
26b2fd9
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Jul 24, 2023
0b50deb
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Aug 22, 2023
bcb8866
Temp disable fast tag syncing for mirrors
harryzcy Aug 22, 2023
80a9877
Refactor release syncing
harryzcy Aug 22, 2023
c535198
Refactor again
harryzcy Aug 23, 2023
6c99fbb
Fix bug
harryzcy Aug 23, 2023
e199b95
Flip bool and add comments
harryzcy Aug 23, 2023
7511099
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Aug 23, 2023
5169422
Make some error messages lowercased
harryzcy Aug 23, 2023
36c2378
Expose context
harryzcy Aug 24, 2023
35ab3bb
No magic literal
harryzcy Aug 24, 2023
4d2cb07
Add comments for OriginalID
harryzcy Aug 24, 2023
6962c14
Update log text
harryzcy Aug 24, 2023
4a33b2a
Revert sync wiki setting check
harryzcy Aug 24, 2023
884cb6d
Default sync items setting to false
harryzcy Aug 24, 2023
775af9e
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Aug 24, 2023
fa40aa7
Expose one more context
harryzcy Aug 24, 2023
2a66750
Update comment
harryzcy Aug 24, 2023
d2ef14c
Plural
harryzcy Aug 24, 2023
9bfd3e3
Expose more context
harryzcy Aug 24, 2023
e39d0ff
Fix a comment issue and early return for prs
harryzcy Aug 24, 2023
ee77675
Fix comment sync error
harryzcy Aug 25, 2023
eacc3ed
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Sep 15, 2023
197489b
Fix lint
harryzcy Sep 16, 2023
049faf8
Somehow git merge didn't delete this file
harryzcy Sep 16, 2023
e28fa78
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Sep 18, 2023
99f9102
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Sep 24, 2023
3b8291f
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Oct 29, 2023
065469c
Fix functions that gets moved in #26969
harryzcy Oct 29, 2023
4886e18
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Nov 4, 2023
218c8f5
Remove unnecessary variables
harryzcy Nov 8, 2023
3103563
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Nov 21, 2023
079814f
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Dec 8, 2023
71dfb67
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Jan 3, 2024
a9247f9
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Jan 14, 2024
079ea08
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Jan 18, 2024
323215f
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Feb 6, 2024
77fd657
Fix build issues from merge
harryzcy Feb 6, 2024
18fffc0
Apply suggestion
harryzcy Feb 26, 2024
a52447b
Merge remote-tracking branch 'upstream/main' into sync-issue-pr-and-more
harryzcy Feb 26, 2024
e677d96
Avoid jquery
harryzcy Feb 26, 2024
db4aa34
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Feb 27, 2024
b143f4a
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Mar 2, 2024
e8119df
Fix build errors
harryzcy Mar 2, 2024
df0a535
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Mar 9, 2024
e2aaa1a
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Apr 26, 2024
cee300a
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Apr 27, 2024
d706aa5
Change `interface{}` to `any`
harryzcy Apr 27, 2024
8211411
Merge branch 'main' into sync-issue-pr-and-more
harryzcy Apr 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cmd/admin.go
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,8 @@ func runRepoSyncReleases(_ *cli.Context) error {
}
log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum)

if err = repo_module.SyncReleasesWithTags(repo, gitRepo); err != nil {
tagOnlyReleases := false
if err = repo_module.SyncReleasesWithTags(repo, gitRepo, tagOnlyReleases); err != nil {
log.Warn(" SyncReleasesWithTags: %v", err)
gitRepo.Close()
continue
Expand Down
1 change: 1 addition & 0 deletions models/issues/comment.go
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,7 @@ type Comment struct {
Poster *user_model.User `xorm:"-"`
OriginalAuthor string
OriginalAuthorID int64
OriginalID int64 // Only used in synced comments
IssueID int64 `xorm:"INDEX"`
Issue *Issue `xorm:"-"`
LabelID int64
Expand Down
100 changes: 92 additions & 8 deletions models/issues/label.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@ type Label struct {
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`

OriginalID int64 // Only for migrating data from other system, used for syncing

NumOpenIssues int `xorm:"-"`
NumOpenRepoIssues int64 `xorm:"-"`
IsChecked bool `xorm:"-"`
Expand Down Expand Up @@ -221,6 +223,10 @@ func NewLabels(labels ...*Label) error {

// UpdateLabel updates label information.
func UpdateLabel(l *Label) error {
return updateLabel(db.DefaultContext, l)
harryzcy marked this conversation as resolved.
Show resolved Hide resolved
}

func updateLabel(ctx context.Context, l *Label) error {
color, err := label.NormalizeColor(l.Color)
if err != nil {
return err
Expand All @@ -232,19 +238,27 @@ func UpdateLabel(l *Label) error {

// DeleteLabel delete a label
func DeleteLabel(id, labelID int64) error {
l, err := GetLabelByID(db.DefaultContext, labelID)
ctx, committer, err := db.TxContext(db.DefaultContext)
if err != nil {
if IsErrLabelNotExist(err) {
return nil
}
return err
}
defer committer.Close()

ctx, committer, err := db.TxContext(db.DefaultContext)
if err = deleteLabel(ctx, id, labelID); err != nil {
return err
}

return committer.Commit()
harryzcy marked this conversation as resolved.
Show resolved Hide resolved
}

func deleteLabel(ctx context.Context, id, labelID int64) error {
l, err := GetLabelByID(ctx, labelID)
if err != nil {
if IsErrLabelNotExist(err) {
return nil
}
return err
}
defer committer.Close()

sess := db.GetEngine(ctx)

Expand All @@ -257,7 +271,9 @@ func DeleteLabel(id, labelID int64) error {

if _, err = sess.ID(labelID).Delete(new(Label)); err != nil {
return err
} else if _, err = sess.
}

if _, err = sess.
Where("label_id = ?", labelID).
Delete(new(IssueLabel)); err != nil {
return err
Expand All @@ -268,7 +284,7 @@ func DeleteLabel(id, labelID int64) error {
return err
}

return committer.Commit()
return nil
}

// GetLabelByID returns a label by given ID.
Expand Down Expand Up @@ -396,6 +412,74 @@ func GetLabelsByRepoID(ctx context.Context, repoID int64, sortType string, listO
return labels, sess.Find(&labels)
}

// UpdateLabels adds, updates, and deletes relevant labels for the given repository.
func UpdateLabelsByRepoID(repoID int64, labels ...*Label) error {
ctx, committer, err := db.TxContext(db.DefaultContext)
if err != nil {
return err
}
defer committer.Close()

existingLabels, err := GetLabelsByRepoID(ctx, repoID, "", db.ListOptions{})
if err != nil {
return err
}
labelsToAdd := make([]*Label, 0)
labelsToUpdate := make([]*Label, 0)
labelsToDelete := make([]*Label, 0)

for _, l := range labels {
var foundLabel *Label
for _, existingLabel := range existingLabels {
if existingLabel.OriginalID == l.OriginalID {
foundLabel = existingLabel
break
}
}

if foundLabel == nil {
labelsToAdd = append(labelsToAdd, l)
} else if foundLabel.Name != l.Name || foundLabel.Description != l.Description ||
foundLabel.Color != l.Color {
l.RepoID = repoID
labelsToUpdate = append(labelsToUpdate, l)
}
}

for _, existingLabel := range existingLabels {
found := false
for _, label := range labels {
if label.OriginalID == existingLabel.OriginalID {
found = true
break
}
}
if !found {
labelsToDelete = append(labelsToDelete, existingLabel)
}
}

for _, l := range labelsToAdd {
if err = NewLabel(ctx, l); err != nil {
return err
}
}

for _, l := range labelsToUpdate {
if err = updateLabel(ctx, l); err != nil {
return err
}
}

for _, l := range labelsToDelete {
if err = deleteLabel(ctx, repoID, l.ID); err != nil {
return err
}
}

return committer.Commit()
}

// CountLabelsByRepoID count number of all labels that belong to given repository by ID.
func CountLabelsByRepoID(repoID int64) (int64, error) {
return db.GetEngine(db.DefaultContext).Where("repo_id = ?", repoID).Count(&Label{})
Expand Down
2 changes: 2 additions & 0 deletions models/issues/milestone.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ type Milestone struct {
DeadlineString string `xorm:"-"`

TotalTrackedTime int64 `xorm:"-"`

OriginalID int64 // ID from the upstream syncing source
}

func init() {
Expand Down
120 changes: 109 additions & 11 deletions models/issues/review.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ type Review struct {
ReviewerTeam *organization.Team `xorm:"-"`
OriginalAuthor string
OriginalAuthorID int64
OriginalID int64
Issue *Issue `xorm:"-"`
IssueID int64 `xorm:"index"`
Content string `xorm:"TEXT"`
Expand Down Expand Up @@ -531,17 +532,7 @@ func InsertReviews(reviews []*Review) error {
return err
}

if _, err := sess.NoAutoTime().Insert(&Comment{
Type: CommentTypeReview,
Content: review.Content,
PosterID: review.ReviewerID,
OriginalAuthor: review.OriginalAuthor,
OriginalAuthorID: review.OriginalAuthorID,
IssueID: review.IssueID,
ReviewID: review.ID,
CreatedUnix: review.CreatedUnix,
UpdatedUnix: review.UpdatedUnix,
}); err != nil {
if _, err := sess.NoAutoTime().Insert(generateCommentFromReview(review)); err != nil {
return err
}

Expand All @@ -559,6 +550,113 @@ func InsertReviews(reviews []*Review) error {
return committer.Commit()
}

// UpsertReviews inserts new reviews and updates existing ones.
// This function is used for syncing from the pull mirror.
func UpsertReviews(reviews []*Review) error {
ctx, committer, err := db.TxContext(db.DefaultContext)
if err != nil {
return err
}
defer committer.Close()
sess := db.GetEngine(ctx)

for _, review := range reviews {
exists, err := sess.Where("original_id = ?", review.OriginalID).Exist(&Review{})
if err != nil {
return err
}

if !exists {
if _, err := sess.NoAutoTime().Insert(review); err != nil {
return err
}

if _, err := sess.NoAutoTime().Insert(generateCommentFromReview(review)); err != nil {
return err
}

for _, c := range review.Comments {
c.ReviewID = review.ID
}

if len(review.Comments) > 0 {
if _, err := sess.NoAutoTime().Insert(review.Comments); err != nil {
return err
}
}
} else {
if _, err = sess.NoAutoTime().Where("original_id = ?", review.OriginalID).Update(review); err != nil {
return err
}

// Get id of the review
if err = sess.NoAutoTime().Where("original_id = ?", review.OriginalID).Find(review); err != nil {
return err
}

comment := generateCommentFromReview(review)
exists, err := existsCommentByReviewIDAndCreatedUnix(sess, comment)
if err != nil {
return err
}

if !exists {
if _, err := sess.NoAutoTime().Insert(comment); err != nil {
return err
}
} else {
if _, err := sess.NoAutoTime().Where("original_id = ?", comment.OriginalID).Update(comment); err != nil {
return err
}
}

for _, c := range review.Comments {
c.ReviewID = review.ID
}

if len(review.Comments) > 0 {
for _, comment := range review.Comments {
exists, err := existsCommentByReviewIDAndCreatedUnix(sess, comment)
if err != nil {
return err
}

if !exists {
if _, err := sess.NoAutoTime().Insert(comment); err != nil {
return err
}
} else {
if _, err := sess.NoAutoTime().Where("original_id = ?", comment.OriginalID).Update(comment); err != nil {
return err
}
}
}
}
}
}

return committer.Commit()
}

func existsCommentByReviewIDAndCreatedUnix(sess db.Engine, comment *Comment) (bool, error) {
return sess.Where("original_id = ?", comment.OriginalID).Exist(&Comment{})
}

func generateCommentFromReview(review *Review) *Comment {
return &Comment{
Type: CommentTypeReview,
Content: review.Content,
PosterID: review.ReviewerID,
OriginalAuthor: review.OriginalAuthor,
OriginalAuthorID: review.OriginalAuthorID,
IssueID: review.IssueID,
ReviewID: review.ID,
CreatedUnix: review.CreatedUnix,
UpdatedUnix: review.UpdatedUnix,
OriginalID: review.OriginalID,
}
}

// AddReviewRequest add a review request from one reviewer
func AddReviewRequest(ctx context.Context, issue *Issue, reviewer, doer *user_model.User) (*Comment, error) {
ctx, committer, err := db.TxContext(ctx)
Expand Down
Loading