Merge branch 'main' into simplify-registry-build

Makefile

@@ -165,7 +165,6 @@ GOIMAGEBUILD_CORE=$(GOIMAGEBUILDCMD) $(GOFLAGS) ${GOTAGS} --ldflags "-w -s $(COR
 GOBUILDPATH_CORE=$(GOBUILDPATHINCONTAINER)/src/core
 GOBUILDPATH_JOBSERVICE=$(GOBUILDPATHINCONTAINER)/src/jobservice
 GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATHINCONTAINER)/src/registryctl
-GOBUILDPATH_MIGRATEPATCH=$(GOBUILDPATHINCONTAINER)/src/cmd/migrate-patch
 GOBUILDPATH_STANDALONE_DB_MIGRATOR=$(GOBUILDPATHINCONTAINER)/src/cmd/standalone-db-migrator
 GOBUILDPATH_EXPORTER=$(GOBUILDPATHINCONTAINER)/src/cmd/exporter
 GOBUILDMAKEPATH=make
@@ -182,7 +181,6 @@ JOBSERVICEBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_JOBSERVICE)
 JOBSERVICEBINARYNAME=harbor_jobservice
 REGISTRYCTLBINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_REGISTRYCTL)
 REGISTRYCTLBINARYNAME=harbor_registryctl
-MIGRATEPATCHBINARYNAME=migrate-patch
 STANDALONE_DB_MIGRATOR_BINARYPATH=$(BUILDPATH)/$(GOBUILDMAKEPATH_STANDALONE_DB_MIGRATOR)
 STANDALONE_DB_MIGRATOR_BINARYNAME=migrate
 
@@ -548,7 +546,6 @@ cleanbinary:
 	if [ -f $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ] ; then rm $(CORE_BINARYPATH)/$(CORE_BINARYNAME) ; fi
 	if [ -f $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ] ; then rm $(JOBSERVICEBINARYPATH)/$(JOBSERVICEBINARYNAME) ; fi
 	if [ -f $(REGISTRYCTLBINARYPATH)/$(REGISTRYCTLBINARYNAME) ] ; then rm $(REGISTRYCTLBINARYPATH)/$(REGISTRYCTLBINARYNAME) ; fi
-	if [ -f $(MIGRATEPATCHBINARYPATH)/$(MIGRATEPATCHBINARYNAME) ] ; then rm $(MIGRATEPATCHBINARYPATH)/$(MIGRATEPATCHBINARYNAME) ; fi
 	rm -rf make/photon/*/binary/
 
 cleanbaseimage:

api/v2.0/swagger.yaml

@@ -109,7 +109,7 @@ paths:
       operationId: searchLdapUser
       summary: Search available ldap users.
       description: |
-        This endpoint searches the available ldap users based on related configuration parameters. Support searched by input ladp configuration, load configuration from the system and specific filter.
+        This endpoint searches the available ldap users based on related configuration parameters. Support searched by input ldap configuration, load configuration from the system and specific filter.
       parameters:
         - $ref: '#/parameters/requestId'
         - name: username
@@ -7340,6 +7340,10 @@ definitions:
         type: string
         description: 'The ID of the tag retention policy for the project'
         x-nullable: true
+      proxy_speed_kb:
+        type: string
+        description: 'The bandwidth limit of proxy cache, in Kbps (kilobits per second). It limits the communication between Harbor and the upstream registry, not the client and the Harbor.'
+        x-nullable: true
   ProjectSummary:
     type: object
     properties:
@@ -7842,6 +7846,12 @@ definitions:
         type: array
         items:
           $ref: '#/definitions/RobotPermission'
+      creator_type:
+        type: string
+        description: The type of the robot creator, like local(harbor_user) or robot.
+      creator_ref:
+        type: integer
+        description: The reference of the robot creator, like the id of harbor user.
       creation_time:
         type: string
         format: date-time
@@ -8985,6 +8995,9 @@ definitions:
       ldap_group_search_scope:
         $ref: '#/definitions/IntegerConfigItem'
         description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE''
+      ldap_group_attach_parallel:
+        $ref: '#/definitions/BoolConfigItem'
+        description: Attach LDAP user group information in parallel.
       ldap_scope:
         $ref: '#/definitions/IntegerConfigItem'
         description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'
@@ -9175,6 +9188,11 @@ definitions:
         description: The scope to search ldap group. ''0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE''
         x-omitempty: true
         x-isnullable: true
+      ldap_group_attach_parallel:
+        type: boolean
+        description: Attach LDAP user group information in parallel, the parallel worker count is 5
+        x-omitempty: true
+        x-isnullable: true
       ldap_scope:
         type: integer
         description: The scope to search ldap users,'0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'

make/migrations/postgresql/0150_2.12.0_schema.up.sql

@@ -0,0 +1,5 @@
+/*
+Add new column creator_ref and creator_type for robot table to record the creator information of the robot
+*/
+ALTER TABLE robot ADD COLUMN IF NOT EXISTS creator_ref integer default 0;
+ALTER TABLE robot ADD COLUMN IF NOT EXISTS creator_type varchar(255);

make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja

@@ -1,4 +1,3 @@
-version: '2.3'
 services:
   log:
     image: goharbor/harbor-log:{{version}}

make/photon/prepare/templates/nginx/nginx.http.conf.jinja

@@ -101,6 +101,9 @@ http {
 
       proxy_buffering off;
       proxy_request_buffering off;
+
+      proxy_send_timeout 900;
+      proxy_read_timeout 900;
     }
 
     location /api/ {

src/common/api/base.go

@@ -116,9 +116,9 @@ func (b *BaseAPI) DecodeJSONReqAndValidate(v interface{}) (bool, error) {
 }
 
 // Redirect does redirection to resource URI with http header status code.
-func (b *BaseAPI) Redirect(statusCode int, resouceID string) {
+func (b *BaseAPI) Redirect(statusCode int, resourceID string) {
 	requestURI := b.Ctx.Request.RequestURI
-	resourceURI := requestURI + "/" + resouceID
+	resourceURI := requestURI + "/" + resourceID
 
 	b.Ctx.Redirect(statusCode, resourceURI)
 }
@@ -138,7 +138,7 @@ func (b *BaseAPI) GetIDFromURL() (int64, error) {
 	return id, nil
 }
 
-// SetPaginationHeader set"Link" and "X-Total-Count" header for pagination request
+// SetPaginationHeader set "Link" and "X-Total-Count" header for pagination request
 func (b *BaseAPI) SetPaginationHeader(total, page, pageSize int64) {
 	b.Ctx.ResponseWriter.Header().Set("X-Total-Count", strconv.FormatInt(total, 10))
 

src/common/const.go

@@ -134,6 +134,7 @@ const (
 	OIDCGroupType                     = 3
 	LDAPGroupAdminDn                  = "ldap_group_admin_dn"
 	LDAPGroupMembershipAttribute      = "ldap_group_membership_attribute"
+	LDAPGroupAttachParallel           = "ldap_group_attach_parallel"
 	DefaultRegistryControllerEndpoint = "http://registryctl:8080"
 	DefaultPortalURL                  = "http://portal:8080"
 	DefaultRegistryCtlURL             = "http://registryctl:8080"
@@ -151,7 +152,7 @@ const (
 	OIDCCallbackPath = "/c/oidc/callback"
 	OIDCLoginPath    = "/c/oidc/login"
 
-	AuthProxyRediretPath = "/c/authproxy/redirect"
+	AuthProxyRedirectPath = "/c/authproxy/redirect"
 
 	// Global notification enable configuration
 	NotificationEnable = "notification_enable"

src/common/http/tls.go

@@ -48,7 +48,7 @@ func GetInternalCertPair() (tls.Certificate, error) {
 
 // GetInternalTLSConfig return a tls.Config for internal https communicate
 func GetInternalTLSConfig() (*tls.Config, error) {
-	// genrate key pair
+	// generate key pair
 	cert, err := GetInternalCertPair()
 	if err != nil {
 		return nil, fmt.Errorf("internal TLS enabled but can't get cert file %w", err)

src/common/job/client.go

@@ -151,7 +151,7 @@ func (d *DefaultClient) SubmitJob(jd *models.JobData) (string, error) {
 	return stats.Stats.JobID, nil
 }
 
-// GetJobLog call jobserivce API to get the log of a job.  It only accepts the UUID of the job
+// GetJobLog call jobservice API to get the log of a job.  It only accepts the UUID of the job
 func (d *DefaultClient) GetJobLog(uuid string) ([]byte, error) {
 	url := d.endpoint + "/api/v1/jobs/" + uuid + "/log"
 	req, err := http.NewRequest(http.MethodGet, url, nil)

src/common/job/models/models.go

@@ -62,15 +62,15 @@ type StatsInfo struct {
 	UpstreamJobID string     `json:"upstream_job_id,omitempty"`   // Ref the upstream job if existing
 	NumericPID    int64      `json:"numeric_policy_id,omitempty"` // The numeric policy ID of the periodic job
 	Parameters    Parameters `json:"parameters,omitempty"`
-	Revision      int64      `json:"revision,omitempty"` // For differentiating the each retry of the same job
+	Revision      int64      `json:"revision,omitempty"` // For differentiating each retry of the same job
 }
 
 // JobPoolStats represents the healthy and status of all the running worker pools.
 type JobPoolStats struct {
 	Pools []*JobPoolStatsData `json:"worker_pools"`
 }
 
-// JobPoolStatsData represent the healthy and status of the worker worker.
+// JobPoolStatsData represent the healthy and status of the worker.
 type JobPoolStatsData struct {
 	WorkerPoolID string   `json:"worker_pool_id"`
 	StartedAt    int64    `json:"started_at"`

src/common/models/job.go

@@ -29,7 +29,7 @@ const (
 	JobCanceled string = "canceled"
 	// JobRetrying indicate the job needs to be retried, it will be scheduled to the end of job queue by statemachine after an interval.
 	JobRetrying string = "retrying"
-	// JobContinue is the status returned by statehandler to tell statemachine to move to next possible state based on trasition table.
+	// JobContinue is the status returned by statehandler to tell statemachine to move to next possible state based on transition table.
 	JobContinue string = "_continue"
 	// JobScheduled ...
 	JobScheduled string = "scheduled"

src/common/models/uaa.go

@@ -14,7 +14,7 @@
 
 package models
 
-// UAASettings wraps the configuraations to access UAA service
+// UAASettings wraps the configurations to access UAA service
 type UAASettings struct {
 	Endpoint     string
 	ClientID     string

src/common/rbac/project/evaluator_test.go

@@ -87,8 +87,8 @@ func TestProjectRoleAccess(t *testing.T) {
 			Username: "username",
 		}
 		evaluator := NewEvaluator(ctl, NewBuilderForUser(user, ctl))
-		resorce := NewNamespace(public.ProjectID).Resource(rbac.ResourceRepository)
-		assert.True(evaluator.HasPermission(context.TODO(), resorce, rbac.ActionPush))
+		resource := NewNamespace(public.ProjectID).Resource(rbac.ResourceRepository)
+		assert.True(evaluator.HasPermission(context.TODO(), resource, rbac.ActionPush))
 	}
 
 	{
@@ -101,8 +101,8 @@ func TestProjectRoleAccess(t *testing.T) {
 			Username: "username",
 		}
 		evaluator := NewEvaluator(ctl, NewBuilderForUser(user, ctl))
-		resorce := NewNamespace(public.ProjectID).Resource(rbac.ResourceRepository)
-		assert.False(evaluator.HasPermission(context.TODO(), resorce, rbac.ActionPush))
+		resource := NewNamespace(public.ProjectID).Resource(rbac.ResourceRepository)
+		assert.False(evaluator.HasPermission(context.TODO(), resource, rbac.ActionPush))
 	}
 }
 

src/common/secret/request.go

@@ -25,7 +25,7 @@ import (
 const HeaderPrefix = "Harbor-Secret "
 
 // FromRequest tries to get Harbor Secret from request header.
-// It will return empty string if the reqeust is nil.
+// It will return empty string if the request is nil.
 func FromRequest(req *http.Request) string {
 	if req == nil {
 		return ""

src/common/utils/email/mail.go

@@ -70,7 +70,7 @@ func Send(addr, identity, username, password string,
 
 // Ping tests the connection and authentication with email server
 // If tls is true, a secure connection is established, or Ping
-// trys to upgrate the insecure connection to a secure one if
+// trys to upgrade the insecure connection to a secure one if
 // email server supports it.
 // Ping doesn't verify the server's certificate and hostname when
 // needed if the parameter insecure is ture
@@ -119,7 +119,7 @@ func newClient(addr, identity, username, password string,
 		return nil, err
 	}
 
-	// try to swith to SSL/TLS
+	// try to switch to SSL/TLS
 	if !tls {
 		if ok, _ := client.Extension("STARTTLS"); ok {
 			log.Debugf("switching the connection with %s to SSL/TLS ...", addr)

src/common/utils/email/mail_test.go

@@ -38,7 +38,7 @@ func TestSend(t *testing.T) {
 	err := Send(addr, identity, username, password,
 		timeout, tls, insecure, from, to,
 		subject, message)
-	// bypass the check due to securty policy change on gmail
+	// bypass the check due to security policy change on gmail
 	// TODO
 	// assert.Nil(t, err)
 
@@ -78,7 +78,7 @@ func TestPing(t *testing.T) {
 	// tls connection
 	err := Ping(addr, identity, username, password,
 		timeout, tls, insecure)
-	// bypass the check due to securty policy change on gmail
+	// bypass the check due to security policy change on gmail
 	// TODO
 	// assert.Nil(t, err)
 

src/common/utils/encrypt.go

@@ -46,8 +46,8 @@ var HashAlg = map[string]func() hash.Hash{
 }
 
 // Encrypt encrypts the content with salt
-func Encrypt(content string, salt string, encrptAlg string) string {
-	return fmt.Sprintf("%x", pbkdf2.Key([]byte(content), []byte(salt), 4096, 16, HashAlg[encrptAlg]))
+func Encrypt(content string, salt string, encryptAlg string) string {
+	return fmt.Sprintf("%x", pbkdf2.Key([]byte(content), []byte(salt), 4096, 16, HashAlg[encryptAlg]))
 }
 
 // ReversibleEncrypt encrypts the str with aes/base64

src/common/utils/passports.go

@@ -72,7 +72,7 @@ func (p *passportsPool) Revoke() bool {
 type LimitedConcurrentRunner interface {
 	// AddTask adds a task to run
 	AddTask(task func() error)
-	// Wait waits all the tasks to be finished, returns error if the any of the tasks gets error
+	// Wait waits all the tasks to be finished, returns error if any of the tasks gets error
 	Wait() (err error)
 	// Cancel cancels all tasks, tasks that already started will continue to run
 	Cancel(err error)
@@ -106,7 +106,7 @@ func (r *limitedConcurrentRunner) AddTask(task func() error) {
 			r.wg.Done()
 		}()
 
-		// Return false means no passport acquired, and no valid passport will be dispatched any more.
+		// Return false means no passport acquired, and no valid passport will be dispatched anymore.
 		// For example, some crucial errors happened and all tasks should be cancelled.
 		if ok := r.passportsPool.Apply(); !ok {
 			return

src/common/utils/test/config.go

@@ -65,7 +65,7 @@ var defaultConfig = map[string]interface{}{
 	common.RobotNamePrefix:            "robot$",
 }
 
-// GetDefaultConfigMap returns the defailt config map for easier modification.
+// GetDefaultConfigMap returns the default config map for easier modification.
 func GetDefaultConfigMap() map[string]interface{} {
 	return defaultConfig
 }