Add force delete runner

apiserver/controllers/instances.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"log"
 	"net/http"
+	"strconv"
 
 	gErrors "github.com/cloudbase/garm-provider-common/errors"
 	"github.com/cloudbase/garm/apiserver/params"
@@ -121,6 +122,12 @@ func (a *APIController) GetInstanceHandler(w http.ResponseWriter, r *http.Reques
 //	    in: path
 //	    required: true
 //
+//	  + name: forceRemove
+//	    description: If true GARM will ignore any provider error when removing the runner and will continue to remove the runner from github and the GARM database.
+//	    type: boolean
+//	    in: query
+//	    required: false
+//
 //	Responses:
 //	  default: APIErrorResponse
 func (a *APIController) DeleteInstanceHandler(w http.ResponseWriter, r *http.Request) {
@@ -138,7 +145,8 @@ func (a *APIController) DeleteInstanceHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if err := a.r.ForceDeleteRunner(ctx, instanceName); err != nil {
+	forceRemove, _ := strconv.ParseBool(r.URL.Query().Get("forceRemove"))
+	if err := a.r.DeleteRunner(ctx, instanceName, forceRemove); err != nil {
 		log.Printf("removing runner: %s", err)
 		handleError(w, err)
 		return

apiserver/swagger.yaml

@@ -597,6 +597,10 @@ paths:
                   name: instanceName
                   required: true
                   type: string
+                - description: If true GARM will ignore any provider error when removing the runner and will continue to remove the runner from github and the GARM database.
+                  in: query
+                  name: forceRemove
+                  type: boolean
             responses:
                 default:
                     description: APIErrorResponse

cmd/garm-cli/cmd/runner.go

@@ -187,12 +187,9 @@ to either cancel the workflow or wait for it to finish.
 			return fmt.Errorf("requires a runner name")
 		}
 
-		if !forceRemove {
-			return fmt.Errorf("use --force-remove-runner=true to remove a runner")
-		}
-
 		deleteInstanceReq := apiClientInstances.NewDeleteInstanceParams()
 		deleteInstanceReq.InstanceName = args[0]
+		deleteInstanceReq.ForceRemove = &forceRemove
 		if err := apiCli.Instances.DeleteInstance(deleteInstanceReq, authToken); err != nil {
 			return err
 		}
@@ -207,7 +204,7 @@ func init() {
 	runnerListCmd.Flags().BoolVarP(&runnerAll, "all", "a", false, "List all runners, regardless of org or repo.")
 	runnerListCmd.MarkFlagsMutuallyExclusive("repo", "org", "enterprise", "all")
 
-	runnerDeleteCmd.Flags().BoolVarP(&forceRemove, "force-remove-runner", "f", false, "Confirm you want to delete a runner")
+	runnerDeleteCmd.Flags().BoolVarP(&forceRemove, "force-remove-runner", "f", false, "Forcefully remove a runner. If set to true, GARM will ignore provider errors when removing the runner.")
 	runnerDeleteCmd.MarkFlagsMutuallyExclusive("force-remove-runner")
 
 	runnerCmd.AddCommand(

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/BurntSushi/toml v1.2.1
-	github.com/cloudbase/garm-provider-common v0.1.0
+	github.com/cloudbase/garm-provider-common v0.1.1-0.20231012061429-49001794e700
 	github.com/go-openapi/errors v0.20.4
 	github.com/go-openapi/runtime v0.26.0
 	github.com/go-openapi/strfmt v0.21.7

go.sum

@@ -22,8 +22,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
 github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04=
 github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cloudbase/garm-provider-common v0.1.0 h1:gc2n8nsLjt7G3InAfqZ+75iZjSIUkIx86d6/DFA2+jc=
-github.com/cloudbase/garm-provider-common v0.1.0/go.mod h1:igxJRT3OlykERYc6ssdRQXcb+BCaeSfnucg6I0OSoDc=
+github.com/cloudbase/garm-provider-common v0.1.1-0.20231012061429-49001794e700 h1:ZCJ1zZ2WI/37ffzpRsu7t5zzShAMThhYsXw7bBNKBR0=
+github.com/cloudbase/garm-provider-common v0.1.1-0.20231012061429-49001794e700/go.mod h1:igxJRT3OlykERYc6ssdRQXcb+BCaeSfnucg6I0OSoDc=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

runner/common/pool.go

@@ -38,22 +38,54 @@ const (
 
 //go:generate mockery --all
 type PoolManager interface {
+	// ID returns the ID of the entity (repo, org, enterprise)
 	ID() string
+	// WebhookSecret returns the unencrypted webhook secret associated with the webhook installed
+	// in GitHub for GARM. For GARM to receive webhook events for an entity, either the operator or
+	// GARM will have to create a webhook in GitHub which points to the GARM API server. To authenticate
+	// the webhook, a webhook secret is used. This function returns that secret.
 	WebhookSecret() string
+	// GithubRunnerRegistrationToken returns a new registration token for a github runner. This is used
+	// for GHES installations that have not yet upgraded to a version >= 3.10. Starting with 3.10, we use
+	// just-in-time runners, which no longer require exposing a runner registration token.
 	GithubRunnerRegistrationToken() (string, error)
+	// HandleWorkflowJob handles a workflow job meant for a particular entity. When a webhook is fired for
+	// a repo, org or enterprise, we determine the destination of that webhook, retrieve the pool manager
+	// for it and call this function with the WorkflowJob as a parameter.
 	HandleWorkflowJob(job params.WorkflowJob) error
+	// RefreshState allows us to update webhook secrets and configuration for a pool manager.
 	RefreshState(param params.UpdatePoolStateParams) error
+	// ForceDeleteRunner will attempt to remove a runner from the pool.
+	//
+	// Deprecated: FunctionName is deprecated. Use DeleteRunner instead.
 	ForceDeleteRunner(runner params.Instance) error
 
+	// DeleteRunner will attempt to remove a runner from the pool. If forceRemove is true, any error
+	// received from the provider will be ignored and we will procede to remove the runner from the database.
+	// An error received while attempting to remove from GitHub (other than 404) will still stop the deletion
+	// process. This can happen if the runner is already processing a job. At which point, you can simply cancel
+	// the job in github. Doing so will prompt GARM to reap the runner automatically.
+	DeleteRunner(runner params.Instance, forceRemove bool) error
+
+	// InstallWebhook will create a webhook in github for the entity associated with this pool manager.
 	InstallWebhook(ctx context.Context, param params.InstallWebhookParams) (params.HookInfo, error)
+	// GetWebhookInfo will return information about the webhook installed in github for the entity associated
 	GetWebhookInfo(ctx context.Context) (params.HookInfo, error)
+	// UninstallWebhook will remove the webhook installed in github for the entity associated with this pool manager.
 	UninstallWebhook(ctx context.Context) error
 
+	// RootCABundle will return a CA bundle that must be installed on all runners in order to properly validate
+	// x509 certificates used by various systems involved. This CA bundle is defined in the GARM config file and
+	// can include multiple CA certificates for the GARM api server, GHES server and any provider API endpoint that
+	// may use internal or self signed certificates.
 	RootCABundle() (params.CertificateBundle, error)
 
-	// PoolManager lifecycle functions. Start/stop pool.
+	// Start will start the pool manager and all associated workers.
 	Start() error
+	// Stop will stop the pool manager and all associated workers.
 	Stop() error
+	// Status will return the current status of the pool manager.
 	Status() params.PoolManagerStatus
+	// Wait will block until the pool manager has stopped.
 	Wait() error
 }