Add new API endpoints for push mirrors management (go-gitea#19841)

- Add a new push mirror to specific repository
- Sync now ( send all the changes to the configured push mirrors )
- Get list of all push mirrors of a repository
- Get a push mirror by ID
- Delete push mirror by ID

Signed-off-by: Mohamed Sekour <[email protected]>
Signed-off-by: Andrew Thornton <[email protected]>
Co-authored-by: zeripath <[email protected]>

integrations/mirror_push_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/models/db"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
@@ -47,7 +48,7 @@ func testMirrorPush(t *testing.T, u *url.URL) {
 
 	doCreatePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword)(t)
 
-	mirrors, err := repo_model.GetPushMirrorsByRepoID(srcRepo.ID)
+	mirrors, _, err := repo_model.GetPushMirrorsByRepoID(db.DefaultContext, srcRepo.ID, db.ListOptions{})
 	assert.NoError(t, err)
 	assert.Len(t, mirrors, 1)
 
@@ -72,7 +73,7 @@ func testMirrorPush(t *testing.T, u *url.URL) {
 
 	// Cleanup
 	doRemovePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword, int(mirrors[0].ID))(t)
-	mirrors, err = repo_model.GetPushMirrorsByRepoID(srcRepo.ID)
+	mirrors, _, err = repo_model.GetPushMirrorsByRepoID(db.DefaultContext, srcRepo.ID, db.ListOptions{})
 	assert.NoError(t, err)
 	assert.Len(t, mirrors, 0)
 }

models/repo/pushmirror.go

@@ -5,12 +5,15 @@
 package repo
 
 import (
+	"context"
 	"errors"
 	"time"
 
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/builder"
 )
 
 // ErrPushMirrorNotExist mirror does not exist error
@@ -29,6 +32,25 @@ type PushMirror struct {
 	LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
 	LastError      string             `xorm:"text"`
 }
+type PushMirrorOptions struct {
+	ID         int64
+	RepoID     int64
+	RemoteName string
+}
+
+func (opts *PushMirrorOptions) toConds() builder.Cond {
+	cond := builder.NewCond()
+	if opts.RepoID > 0 {
+		cond = cond.And(builder.Eq{"repo_id": opts.RepoID})
+	}
+	if opts.RemoteName != "" {
+		cond = cond.And(builder.Eq{"remote_name": opts.RemoteName})
+	}
+	if opts.ID > 0 {
+		cond = cond.And(builder.Eq{"id": opts.ID})
+	}
+	return cond
+}
 
 func init() {
 	db.RegisterModel(new(PushMirror))
@@ -53,45 +75,48 @@ func (m *PushMirror) GetRemoteName() string {
 }
 
 // InsertPushMirror inserts a push-mirror to database
-func InsertPushMirror(m *PushMirror) error {
-	_, err := db.GetEngine(db.DefaultContext).Insert(m)
+func InsertPushMirror(ctx context.Context, m *PushMirror) error {
+	_, err := db.GetEngine(ctx).Insert(m)
 	return err
 }
 
 // UpdatePushMirror updates the push-mirror
-func UpdatePushMirror(m *PushMirror) error {
-	_, err := db.GetEngine(db.DefaultContext).ID(m.ID).AllCols().Update(m)
-	return err
-}
-
-// DeletePushMirrorByID deletes a push-mirrors by ID
-func DeletePushMirrorByID(ID int64) error {
-	_, err := db.GetEngine(db.DefaultContext).ID(ID).Delete(&PushMirror{})
+func UpdatePushMirror(ctx context.Context, m *PushMirror) error {
+	_, err := db.GetEngine(ctx).ID(m.ID).AllCols().Update(m)
 	return err
 }
 
-// DeletePushMirrorsByRepoID deletes all push-mirrors by repoID
-func DeletePushMirrorsByRepoID(repoID int64) error {
-	_, err := db.GetEngine(db.DefaultContext).Delete(&PushMirror{RepoID: repoID})
-	return err
+func DeletePushMirrors(ctx context.Context, opts PushMirrorOptions) error {
+	if opts.RepoID > 0 {
+		_, err := db.GetEngine(ctx).Where(opts.toConds()).Delete(&PushMirror{})
+		return err
+	}
+	return errors.New("repoID required and must be set")
 }
 
-// GetPushMirrorByID returns push-mirror information.
-func GetPushMirrorByID(ID int64) (*PushMirror, error) {
-	m := &PushMirror{}
-	has, err := db.GetEngine(db.DefaultContext).ID(ID).Get(m)
+func GetPushMirror(ctx context.Context, opts PushMirrorOptions) (*PushMirror, error) {
+	mirror := &PushMirror{}
+	exist, err := db.GetEngine(ctx).Where(opts.toConds()).Get(mirror)
 	if err != nil {
 		return nil, err
-	} else if !has {
+	} else if !exist {
 		return nil, ErrPushMirrorNotExist
 	}
-	return m, nil
+	return mirror, nil
 }
 
 // GetPushMirrorsByRepoID returns push-mirror information of a repository.
-func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) {
+func GetPushMirrorsByRepoID(ctx context.Context, repoID int64, listOptions db.ListOptions) ([]*PushMirror, int64, error) {
+	sess := db.GetEngine(ctx).Where("repo_id = ?", repoID)
+	if listOptions.Page != 0 {
+		sess = db.SetSessionPagination(sess, &listOptions)
+		mirrors := make([]*PushMirror, 0, listOptions.PageSize)
+		count, err := sess.FindAndCount(&mirrors)
+		return mirrors, count, err
+	}
 	mirrors := make([]*PushMirror, 0, 10)
-	return mirrors, db.GetEngine(db.DefaultContext).Where("repo_id=?", repoID).Find(&mirrors)
+	count, err := sess.FindAndCount(&mirrors)
+	return mirrors, count, err
 }
 
 // GetPushMirrorsSyncedOnCommit returns push-mirrors for this repo that should be updated by new commits
@@ -103,8 +128,8 @@ func GetPushMirrorsSyncedOnCommit(repoID int64) ([]*PushMirror, error) {
 }
 
 // PushMirrorsIterate iterates all push-mirror repositories.
-func PushMirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
-	return db.GetEngine(db.DefaultContext).
+func PushMirrorsIterate(ctx context.Context, limit int, f func(idx int, bean interface{}) error) error {
+	return db.GetEngine(ctx).
 		Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()).
 		And("`interval` != 0").
 		OrderBy("last_update ASC").

models/repo/pushmirror_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"code.gitea.io/gitea/models/db"
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unittest"
 	"code.gitea.io/gitea/modules/timeutil"
@@ -20,28 +21,28 @@ func TestPushMirrorsIterate(t *testing.T) {
 
 	now := timeutil.TimeStampNow()
 
-	repo_model.InsertPushMirror(&repo_model.PushMirror{
+	repo_model.InsertPushMirror(db.DefaultContext, &repo_model.PushMirror{
 		RemoteName:     "test-1",
 		LastUpdateUnix: now,
 		Interval:       1,
 	})
 
 	long, _ := time.ParseDuration("24h")
-	repo_model.InsertPushMirror(&repo_model.PushMirror{
+	repo_model.InsertPushMirror(db.DefaultContext, &repo_model.PushMirror{
 		RemoteName:     "test-2",
 		LastUpdateUnix: now,
 		Interval:       long,
 	})
 
-	repo_model.InsertPushMirror(&repo_model.PushMirror{
+	repo_model.InsertPushMirror(db.DefaultContext, &repo_model.PushMirror{
 		RemoteName:     "test-3",
 		LastUpdateUnix: now,
 		Interval:       0,
 	})
 
 	time.Sleep(1 * time.Millisecond)
 
-	repo_model.PushMirrorsIterate(1, func(idx int, bean interface{}) error {
+	repo_model.PushMirrorsIterate(db.DefaultContext, 1, func(idx int, bean interface{}) error {
 		m, ok := bean.(*repo_model.PushMirror)
 		assert.True(t, ok)
 		assert.Equal(t, "test-1", m.RemoteName)

modules/context/repo.go

@@ -393,7 +393,7 @@ func repoAssignment(ctx *Context, repo *repo_model.Repository) {
 		}
 	}
 
-	pushMirrors, err := repo_model.GetPushMirrorsByRepoID(repo.ID)
+	pushMirrors, _, err := repo_model.GetPushMirrorsByRepoID(ctx, repo.ID, db.ListOptions{})
 	if err != nil {
 		ctx.ServerError("GetPushMirrorsByRepoID", err)
 		return

modules/convert/mirror.go

@@ -0,0 +1,39 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package convert
+
+import (
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/git"
+	api "code.gitea.io/gitea/modules/structs"
+)
+
+// ToPushMirror convert from repo_model.PushMirror and remoteAddress to api.TopicResponse
+func ToPushMirror(pm *repo_model.PushMirror) (*api.PushMirror, error) {
+	repo := pm.GetRepository()
+	remoteAddress, err := getRemoteAddress(repo, pm.RemoteName)
+	if err != nil {
+		return nil, err
+	}
+	return &api.PushMirror{
+		RepoName:       repo.Name,
+		RemoteName:     pm.RemoteName,
+		RemoteAddress:  remoteAddress,
+		CreatedUnix:    pm.CreatedUnix.FormatLong(),
+		LastUpdateUnix: pm.LastUpdateUnix.FormatLong(),
+		LastError:      pm.LastError,
+		Interval:       pm.Interval.String(),
+	}, nil
+}
+
+func getRemoteAddress(repo *repo_model.Repository, remoteName string) (string, error) {
+	url, err := git.GetRemoteURL(git.DefaultContext, repo.RepoPath(), remoteName)
+	if err != nil {
+		return "", err
+	}
+	// remove confidential information
+	url.User = nil
+	return url.String(), nil
+}

modules/structs/mirror.go

@@ -0,0 +1,25 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package structs
+
+// CreatePushMirrorOption represents need information to create a push mirror of a repository.
+type CreatePushMirrorOption struct {
+	RemoteAddress  string `json:"remote_address"`
+	RemoteUsername string `json:"remote_username"`
+	RemotePassword string `json:"remote_password"`
+	Interval       string `json:"interval"`
+}
+
+// PushMirror represents information of a push mirror
+// swagger:model
+type PushMirror struct {
+	RepoName       string `json:"repo_name"`
+	RemoteName     string `json:"remote_name"`
+	RemoteAddress  string `json:"remote_address"`
+	CreatedUnix    string `json:"created"`
+	LastUpdateUnix string `json:"last_update"`
+	LastError      string `json:"last_error"`
+	Interval       string `json:"interval"`
+}

routers/api/v1/api.go

@@ -982,6 +982,15 @@ func Routes() *web.Route {
 					})
 				}, reqRepoReader(unit.TypeReleases))
 				m.Post("/mirror-sync", reqToken(), reqRepoWriter(unit.TypeCode), repo.MirrorSync)
+				m.Post("/push_mirrors-sync", reqAdmin(), repo.PushMirrorSync)
+				m.Group("/push_mirrors", func() {
+					m.Combo("").Get(repo.ListPushMirrors).
+						Post(bind(api.CreatePushMirrorOption{}), repo.AddPushMirror)
+					m.Combo("/{name}").
+						Delete(repo.DeletePushMirrorByRemoteName).
+						Get(repo.GetPushMirrorByName)
+				}, reqAdmin())
+
 				m.Get("/editorconfig/{filename}", context.ReferencesGitRepo(), context.RepoRefForAPI, reqRepoReader(unit.TypeCode), repo.GetEditorconfig)
 				m.Group("/pulls", func() {
 					m.Combo("").Get(repo.ListPullRequests).