feat: Add tensorboard delete command to CLI

docs/release-notes/tensorboard-delete.rst

@@ -0,0 +1,6 @@
+:orphan:
+
+**New Features**
+
+-  CLI: Add a new CLI command ``det e delete-tb-files [Experiment ID]`` to delete local TensorBoard
+   files associated to a given experiment.

e2e_tests/tests/command/test_tensorboard.py

@@ -223,6 +223,26 @@ def test_tensorboard_inherit_image_pull_secrets() -> None:
     assert ips == exp_secrets, (ips, exp_secrets)
 
 
+@pytest.mark.e2e_cpu
+def test_delete_tensorboard_for_experiment() -> None:
+    """
+    Start a random experiment, start a TensorBoard instance pointed to
+    the experiment, delete tensorboard and verify deletion.
+    """
+    config_obj = conf.load_config(conf.tutorials_path("mnist_pytorch/const.yaml"))
+    experiment_id = exp.run_basic_test_with_temp_config(
+        config_obj, conf.tutorials_path("mnist_pytorch"), 1
+    )
+
+    command = ["det", "e", "delete-tb-files", str(experiment_id)]
+    subprocess.run(command, universal_newlines=True, stdout=subprocess.PIPE, check=True)
+
+    # Check if Tensorboard files are deleted
+    tb_path = sorted(pathlib.Path("/tmp/determined-cp/").glob("*/tensorboard"))[0]
+    tb_path = tb_path / "experiment" / str(experiment_id)
+    assert not pathlib.Path(tb_path).exists()
+
+
 @pytest.mark.e2e_cpu
 def test_tensorboard_directory_storage(tmp_path: pathlib.Path) -> None:
     config_obj = conf.load_config(conf.fixtures_path("no_op/single-one-short-step.yaml"))

harness/determined/cli/experiment.py

@@ -1001,6 +1001,11 @@ def move_experiment(args: Namespace) -> None:
     print(f'Moved experiment {args.experiment_id} to project "{args.project_name}"')
 
 
+@authentication.required
+def delete_tensorboard_files(args: Namespace) -> None:
+    bindings.delete_DeleteTensorboardFiles(cli.setup_session(args), experimentId=args.experiment_id)
+
+
 def none_or_int(string: str) -> Optional[int]:
     if string.lower().strip() in ("null", "none"):
         return None
@@ -1400,6 +1405,14 @@ def experiment_id_arg(help: str) -> Arg:  # noqa: A002
                 ),
             ],
         ),
+        Cmd(
+            "delete-tb-files",
+            delete_tensorboard_files,
+            "delete TensorBoard files associate with the proived experiment ID",
+            [
+                Arg("experiment_id", type=int, help="Experiment ID"),
+            ],
+        ),
     ],
 )
 

master/internal/api_experiment.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	structpb "github.com/golang/protobuf/ptypes/struct"
+	"github.com/google/uuid"
 	"github.com/labstack/echo/v4"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -2969,3 +2970,39 @@ func (a *apiServer) DeleteExperimentLabel(ctx context.Context,
 
 	return &apiv1.DeleteExperimentLabelResponse{Labels: exp.Labels}, nil
 }
+
+func (a *apiServer) DeleteTensorboardFiles(
+	ctx context.Context, req *apiv1.DeleteTensorboardFilesRequest,
+) (resp *apiv1.DeleteTensorboardFilesResponse, err error) {
+	curUser, _, err := grpcutil.GetUser(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	exp, err := db.ExperimentByID(context.TODO(), int(req.ExperimentId))
+	if err != nil {
+		return nil, err
+	}
+
+	workspaceID, err := workspace.WorkspacesIDsByExperimentIDs(ctx, []int{exp.ID})
+	if err != nil {
+		return nil, err
+	}
+	agentUserGroup, err := user.GetAgentUserGroup(context.TODO(), *exp.OwnerID, workspaceID[0])
+	if err != nil {
+		return nil, err
+	}
+
+	var uuidList []uuid.UUID
+	err = runCheckpointGCTask(
+		a.m.rm, a.m.db, model.NewTaskID(), exp.JobID, exp.StartTime, *a.m.taskSpec, exp.ID,
+		exp.Config, uuidList, nil, true, agentUserGroup, curUser,
+		nil,
+	)
+	if err != nil {
+		log.WithError(err).Errorf("failed to gc tensorboard for experiment: %d", exp.ID)
+		return nil, err
+	}
+
+	return &apiv1.DeleteTensorboardFilesResponse{}, nil
+}

master/pkg/tasks/task_gc.go

@@ -106,9 +106,13 @@ func (g GCCkptSpec) ToTaskSpec() TaskSpec {
 		"--experiment-id",
 		strconv.Itoa(g.ExperimentID),
 		"--storage-config", fmt.Sprintf("/run/determined/%s", storageConfigPath),
-		"--delete", fmt.Sprintf("/run/determined/%s", checkpointsToDeletePath),
-		"--globs", fmt.Sprintf("/run/determined/%s", checkpointsGlobsPath),
 	}
+
+	if len(g.ToDelete) > 0 {
+		res.Entrypoint = append(res.Entrypoint, "--delete", fmt.Sprintf("/run/determined/%s", checkpointsToDeletePath))
+		res.Entrypoint = append(res.Entrypoint, "--globs", fmt.Sprintf("/run/determined/%s", checkpointsGlobsPath))
+	}
+
 	if g.DeleteTensorboards {
 		res.Entrypoint = append(res.Entrypoint, "--delete-tensorboards")
 	}