diff --git a/agent/api.go b/agent/api.go
index 0a3cb05a8a..226d489469 100755
--- a/agent/api.go
+++ b/agent/api.go
@@ -11,6 +11,7 @@ type APIClient interface {
 	AcceptJob(*api.Job) (*api.Job, *api.Response, error)
 	AcquireJob(string) (*api.Job, *api.Response, error)
 	Annotate(string, *api.Annotation) (*api.Response, error)
+	AnnotationRemove(string, string) (*api.Response, error)
 	Config() api.Config
 	Connect() (*api.Response, error)
 	CreateArtifacts(string, *api.ArtifactBatch) (*api.ArtifactBatchCreateResponse, *api.Response, error)
diff --git a/api/annotations.go b/api/annotations.go
index be4e060c8d..d078b8daaa 100644
--- a/api/annotations.go
+++ b/api/annotations.go
@@ -8,7 +8,6 @@ type Annotation struct {
 	Context string `json:"context,omitempty"`
 	Style   string `json:"style,omitempty"`
 	Append  bool   `json:"append,omitempty"`
-	Remove  bool   `json:"remove,omitempty"`
 }
 
 // Annotate a build in the Buildkite UI
@@ -22,3 +21,15 @@ func (c *Client) Annotate(jobId string, annotation *Annotation) (*Response, erro
 
 	return c.doRequest(req, nil)
 }
+
+// Remove an annotation from a build
+func (c *Client) AnnotationRemove(jobId string, context string) (*Response, error) {
+	u := fmt.Sprintf("jobs/%s/annotations/%s", jobId, context)
+
+	req, err := c.newRequest("DELETE", u, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return c.doRequest(req, nil)
+}
diff --git a/clicommand/annotate.go b/clicommand/annotate.go
index 6ba2eaec8b..016fb2867e 100644
--- a/clicommand/annotate.go
+++ b/clicommand/annotate.go
@@ -52,7 +52,6 @@ type AnnotateConfig struct {
 	Style   string `cli:"style"`
 	Context string `cli:"context"`
 	Append  bool   `cli:"append"`
-	Remove  bool   `cli:"remove"`
 	Job     string `cli:"job" validate:"required"`
 
 	// Global flags
@@ -88,11 +87,6 @@ var AnnotateCommand = cli.Command{
 			Usage:  "Append to the body of an existing annotation",
 			EnvVar: "BUILDKITE_ANNOTATION_APPEND",
 		},
-		cli.BoolFlag{
-			Name:   "remove",
-			Usage:  "Remove an existing annotation",
-			EnvVar: "BUILDKITE_ANNOTATION_REMOVE",
-		},
 		cli.StringFlag{
 			Name:   "job",
 			Value:  "",
@@ -153,7 +147,6 @@ var AnnotateCommand = cli.Command{
 			Style:   cfg.Style,
 			Context: cfg.Context,
 			Append:  cfg.Append,
-			Remove:  cfg.Remove,
 		}
 
 		// Retry the annotation a few times before giving up
diff --git a/clicommand/annotation_remove.go b/clicommand/annotation_remove.go
new file mode 100644
index 0000000000..4618b174c2
--- /dev/null
+++ b/clicommand/annotation_remove.go
@@ -0,0 +1,121 @@
+package clicommand
+
+import (
+  "time"
+
+  "github.com/buildkite/agent/v3/api"
+  "github.com/buildkite/agent/v3/cliconfig"
+  "github.com/buildkite/agent/v3/retry"
+  "github.com/urfave/cli"
+)
+
+var AnnotationRemoveHelpDescription = `Usage:
+
+   buildkite-agent annotation remove [arguments...]
+
+Description:
+
+   Remove an existing annotation which was previously published using the
+   buildkite-agent annotate command.
+
+   Or if you leave context blank, it will use the default context.
+
+Example:
+
+   $ buildkite-agent annotation remove
+   $ buildkite-agent annotation remove --context "remove-me"`
+
+type AnnotationRemoveConfig struct {
+  Context string `cli:"context" validate:"required"`
+  Job     string `cli:"job" validate:"required"`
+
+  // Global flags
+  Debug   bool         `cli:"debug"`
+  NoColor bool         `cli:"no-color"`
+  Experiments []string `cli:"experiment" normalize:"list"`
+  Profile string       `cli:"profile"`
+
+  // API config
+  DebugHTTP        bool   `cli:"debug-http"`
+  AgentAccessToken string `cli:"agent-access-token" validate:"required"`
+  Endpoint         string `cli:"endpoint" validate:"required"`
+  NoHTTP2          bool   `cli:"no-http2"`
+}
+
+var AnnotationRemoveCommand = cli.Command{
+  Name:        "remove",
+  Usage:       "Remove an existing annotation from a Buildkite build",
+  Description: AnnotationRemoveHelpDescription,
+  Flags: []cli.Flag{
+    cli.StringFlag{
+      Name:   "context",
+      Value:  "default",
+      Usage:  "The context of the annotation used to differentiate this annotation from others",
+      EnvVar: "BUILDKITE_ANNOTATION_CONTEXT",
+    },
+    cli.StringFlag{
+      Name:   "job",
+      Value:  "",
+      Usage:  "Which job should the annotation come from",
+      EnvVar: "BUILDKITE_JOB_ID",
+    },
+
+    // API Flags
+    AgentAccessTokenFlag,
+    EndpointFlag,
+    NoHTTP2Flag,
+    DebugHTTPFlag,
+
+    // Global flags
+    NoColorFlag,
+    DebugFlag,
+    ExperimentsFlag,
+    ProfileFlag,
+  },
+  Action: func(c *cli.Context) {
+    // The configuration will be loaded into this struct
+    cfg := AnnotationRemoveConfig{}
+
+    l := CreateLogger(&cfg)
+
+    // Load the configuration
+    if err := cliconfig.Load(c, l, &cfg); err != nil {
+      l.Fatal("%s", err)
+    }
+
+    // Setup any global configuration options
+    done := HandleGlobalFlags(l, cfg)
+    defer done()
+
+    var err error
+
+    // Create the API client
+    client := api.NewClient(l, loadAPIClientConfig(cfg, `AgentAccessToken`))
+
+    // Retry the removal a few times before giving up
+    err = retry.Do(func(s *retry.Stats) error {
+      // Attempt to remove the annotation
+      resp, err := client.AnnotationRemove(cfg.Job, cfg.Context)
+
+      // Don't bother retrying if the response was one of these statuses
+      if resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 404 || resp.StatusCode == 400) {
+        s.Break()
+        return err
+      }
+
+      // Show the unexpected error
+      if err != nil {
+        l.Warn("%s (%s)", err, s)
+      }
+
+      return err
+    }, &retry.Config{Maximum: 5, Interval: 1 * time.Second, Jitter: true})
+
+    // Show a fatal error if we gave up trying to create the annotation
+    if err != nil {
+      l.Fatal("Failed to remove annotation: %s", err)
+    }
+
+    l.Debug("Successfully removed annotation")
+  },
+}
diff --git a/main.go b/main.go
index 89134dbc52..ef71ab024d 100644
--- a/main.go
+++ b/main.go
@@ -63,6 +63,13 @@ func main() {
 	app.Commands = []cli.Command{
 		clicommand.AgentStartCommand,
 		clicommand.AnnotateCommand,
+		{
+			Name:  "annotation",
+			Usage: "Make changes an annotation on the currently running build",
+			Subcommands: []cli.Command{
+				clicommand.AnnotationRemoveCommand,
+			},
+		},
 		{
 			Name:  "artifact",
 			Usage: "Upload/download artifacts from Buildkite jobs",