Get rid of test artifacts

examples/tensorflow/scripts/distributed_training/horovod_mnist_estimator.py

@@ -186,7 +186,7 @@ def main(unused_argv):
 
     # Download and load MNIST dataset.
     (train_data, train_labels), (eval_data, eval_labels) = keras.datasets.mnist.load_data(
-        "MNIST-data-%d" % hvd.rank()
+        "/tmp/MNIST-data-%d" % hvd.rank()
     )
 
     # The shape of downloaded data is (-1, 28, 28), hence we need to reshape it

examples/tensorflow/scripts/keras.py

@@ -42,7 +42,7 @@ def scale(image, label):
     eval_dataset = mnist_test.map(scale).batch(BATCH_SIZE)
 
     hook = KerasHook(
-        out_dir="~/ts_outputs/",
+        out_dir="/tmp/ts_outputs/",
         include_collections=[
             # CollectionKeys.WEIGHTS,
             # CollectionKeys.GRADIENTS,
@@ -76,7 +76,7 @@ def scale(image, label):
 
     callbacks = [
         hook
-        # tf.keras.callbacks.TensorBoard(log_dir='./logs'),
+        # tf.keras.callbacks.TensorBoard(log_dir='/tmp/logs'),
     ]
 
     model.fit(train_dataset, epochs=1, callbacks=callbacks)

smdebug/core/json_config.py

@@ -3,7 +3,7 @@
 
 {
   "S3Path": "s3://bucket/prefix",
-  "LocalPath": "newlogsRunTest/test_hook_from_json_config_full",
+  "LocalPath": "/tmp/test_hook_from_json_config_full",
   "HookParameters": {
     "export_tensorboard": true,
     "tensorboard_dir": "/tmp/tensorboard",

tests/analysis/trials/test_modes.py

@@ -21,7 +21,7 @@ def test_modes_on_global_data():
 
 def test_mode_data():
     run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-    trial_dir = "ts_outputs/" + run_id
+    trial_dir = "/tmp/ts_outputs/" + run_id
 
     c = CollectionManager()
     c.add("default")
@@ -84,4 +84,4 @@ def test_mode_data():
             assert tr.tensor("arr").value(i, mode=modes.TRAIN) is not None
             assert tr.tensor("arr").value(i, mode=modes.EVAL) is not None
 
-    shutil.rmtree("ts_outputs/" + run_id)
+    shutil.rmtree("/tmp/ts_outputs/" + run_id)

tests/analysis/trials/test_tensors_api.py

@@ -55,7 +55,7 @@ def test_tensors(out_dir):
 
 def test_mode_data():
     run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-    trial_dir = "ts_outputs/" + run_id
+    trial_dir = "/tmp/ts_outputs/" + run_id
 
     c = CollectionManager()
     c.add("default")

tests/core/test_collections.py

@@ -56,9 +56,9 @@ def test_manager_export_load():
     cm.add(Collection("trial1"))
     cm.add("trial2")
     cm.get("trial2").include("total_loss")
-    cm.export("dummy_trial", DEFAULT_COLLECTIONS_FILE_NAME)
+    cm.export("/tmp/dummy_trial", DEFAULT_COLLECTIONS_FILE_NAME)
     cm2 = CollectionManager.load(
-        os.path.join(get_path_to_collections("dummy_trial"), DEFAULT_COLLECTIONS_FILE_NAME)
+        os.path.join(get_path_to_collections("/tmp/dummy_trial"), DEFAULT_COLLECTIONS_FILE_NAME)
     )
     assert cm == cm2
 

tests/core/test_modes.py

@@ -21,7 +21,7 @@ def test_mode_writing():
     run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
     worker = socket.gethostname()
     for s in range(0, 10):
-        fw = FileWriter(trial_dir="ts_outputs/" + run_id, step=s, worker=worker)
+        fw = FileWriter(trial_dir="/tmp/ts_outputs/" + run_id, step=s, worker=worker)
         if s % 2 == 0:
             fw.write_tensor(
                 tdata=np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32),
@@ -37,8 +37,8 @@ def test_mode_writing():
                 mode_step=s // 2,
             )
         fw.close()
-    write_dummy_collection_file("ts_outputs/" + run_id)
-    files = glob.glob("ts_outputs/" + run_id + "/**/*.tfevents", recursive=True)
+    write_dummy_collection_file("/tmp/ts_outputs/" + run_id)
+    files = glob.glob("/tmp/ts_outputs/" + run_id + "/**/*.tfevents", recursive=True)
 
     global_steps = []
     train_steps = []
@@ -56,8 +56,8 @@ def test_mode_writing():
             assert mode_step == step // 2
             global_steps.append(step)
 
-    trial = create_trial("ts_outputs/" + run_id)
+    trial = create_trial("/tmp/ts_outputs/" + run_id)
     assert trial.steps() == sorted(global_steps)
     assert trial.steps(ModeKeys.TRAIN) == sorted(train_steps)
     assert trial.steps(ModeKeys.EVAL) == sorted(eval_steps)
-    shutil.rmtree("ts_outputs/" + run_id)
+    shutil.rmtree("/tmp/ts_outputs/" + run_id)

tests/mxnet/test_hook.py

@@ -16,7 +16,7 @@
 def test_hook():
     save_config = SaveConfig(save_steps=[0, 1, 2, 3])
     run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-    out_dir = "newlogsRunTest/" + run_id
+    out_dir = "/tmp/newlogsRunTest/" + run_id
     hook = t_hook(out_dir=out_dir, save_config=save_config)
     assert has_training_ended(out_dir) == False
     run_mnist_gluon_model(
@@ -26,7 +26,7 @@ def test_hook():
 
 
 def test_hook_from_json_config():
-    out_dir = "newlogsRunTest1/test_hook_from_json_config"
+    out_dir = "/tmp/newlogsRunTest1/test_hook_from_json_config"
     shutil.rmtree(out_dir, True)
     os.environ[
         CONFIG_FILE_PATH_ENV_STR
@@ -40,7 +40,7 @@ def test_hook_from_json_config():
 
 
 def test_hook_from_json_config_full():
-    out_dir = "newlogsRunTest2/test_hook_from_json_config_full"
+    out_dir = "/tmp/newlogsRunTest2/test_hook_from_json_config_full"
     shutil.rmtree(out_dir, True)
     os.environ[
         CONFIG_FILE_PATH_ENV_STR

tests/mxnet/test_hook_all_zero.py

@@ -20,7 +20,7 @@ def test_hook_all_zero(hook=None, out_dir=None):
         hook_created = True
         save_config = SaveConfig(save_steps=[0, 1, 2, 3])
         run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-        out_dir = "./newlogsRunTest/" + run_id
+        out_dir = "/tmp/newlogsRunTest/" + run_id
         print("Registering the hook with out_dir {0}".format(out_dir))
         hook = t_hook(
             out_dir=out_dir,
@@ -52,7 +52,7 @@ def test_hook_all_zero_hook_from_json():
     import shutil
     import os
 
-    out_dir = "newlogsRunTest2/test_hook_all_zero_hook_from_json"
+    out_dir = "/tmp/newlogsRunTest2/test_hook_all_zero_hook_from_json"
     shutil.rmtree(out_dir, True)
     os.environ[
         CONFIG_FILE_PATH_ENV_STR

tests/mxnet/test_hook_custom_collection.py

@@ -13,7 +13,7 @@
 def test_hook_custom_collection():
     save_config = SaveConfig(save_steps=[0, 1, 2, 3])
     run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-    out_dir = "./newlogsRunTest/" + run_id
+    out_dir = "/tmp/" + run_id
     hook = t_hook(out_dir=out_dir, save_config=save_config, include_collections=["ReluActivation"])
     hook.get_collection("ReluActivation").include(["relu*", "input_*"])
     run_mnist_gluon_model(hook=hook, num_steps_train=10, num_steps_eval=10)

tests/mxnet/test_hook_loss_collection.py

@@ -15,7 +15,7 @@
 def test_loss_collection_default():
     save_config = SaveConfig(save_steps=[0, 1, 2, 3])
     run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-    out_dir = "newlogsRunTest/" + run_id
+    out_dir = "/tmp/" + run_id
     hook = t_hook(out_dir=out_dir, save_config=save_config)
     assert has_training_ended(out_dir) == False
     run_mnist_gluon_model(
@@ -39,7 +39,7 @@ def test_loss_collection_default():
 def test_loss_collection_with_no_other_collections():
     save_config = SaveConfig(save_steps=[0, 1, 2, 3])
     run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-    out_dir = "newlogsRunTest/" + run_id
+    out_dir = "/tmp/" + run_id
     hook = t_hook(out_dir=out_dir, save_config=save_config, include_collections=[])
     assert has_training_ended(out_dir) == False
     run_mnist_gluon_model(

tests/mxnet/test_hook_reduce_config.py

@@ -19,7 +19,7 @@ def test_save_config(hook=None, out_dir=None):
         global_save_config = SaveConfig(save_steps=[0, 1, 2, 3])
 
         run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-        out_dir = "./newlogsRunTest/" + run_id
+        out_dir = "/tmp/newlogsRunTest/" + run_id
         print("Registering the hook with out_dir {0}".format(out_dir))
         hook = t_hook(
             out_dir=out_dir,
@@ -86,7 +86,7 @@ def test_save_config_hook_from_json():
     from smdebug.core.json_config import CONFIG_FILE_PATH_ENV_STR
     import os
 
-    out_dir = "newlogsRunTest2/test_hook_reduce_config_hook_from_json"
+    out_dir = "/tmp/newlogsRunTest2/test_hook_reduce_config_hook_from_json"
     shutil.rmtree(out_dir, True)
     os.environ[
         CONFIG_FILE_PATH_ENV_STR

tests/mxnet/test_hook_save_all.py

@@ -17,7 +17,7 @@ def test_save_all(hook=None, out_dir=None):
         hook_created = True
         save_config = SaveConfig(save_steps=[0, 1, 2, 3])
         run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-        out_dir = "./newlogsRunTest/" + run_id
+        out_dir = "/tmp/" + run_id
         print("Registering the hook with out_dir {}".format(out_dir))
         hook = t_hook(out_dir=out_dir, save_config=save_config, save_all=True)
     run_mnist_gluon_model(hook=hook, num_steps_train=7, num_steps_eval=5)
@@ -39,7 +39,7 @@ def test_save_all_hook_from_json():
     from smdebug.core.json_config import CONFIG_FILE_PATH_ENV_STR
     import os
 
-    out_dir = "newlogsRunTest2/test_hook_save_all_hook_from_json"
+    out_dir = "/tmp/newlogsRunTest2/test_hook_save_all_hook_from_json"
     shutil.rmtree(out_dir, True)
     os.environ[
         CONFIG_FILE_PATH_ENV_STR

tests/mxnet/test_hook_save_config.py

@@ -15,7 +15,7 @@ def test_save_config(hook=None):
     if hook is None:
         save_config_collection = SaveConfig(save_steps=[4, 5, 6])
         run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-        out_dir = "./newlogsRunTest/" + run_id
+        out_dir = "/tmp/" + run_id
         save_config = SaveConfig(save_steps=[0, 1, 2, 3])
         hook = t_hook(
             out_dir=out_dir,
@@ -34,7 +34,7 @@ def test_save_config(hook=None):
 def test_save_config_hookjson_config():
     from smdebug.core.json_config import CONFIG_FILE_PATH_ENV_STR
 
-    out_dir = "newlogsRunTest2/test_hook_from_json_config_full"
+    out_dir = "/tmp/test_hook_from_json_config_full"
     shutil.rmtree(out_dir, True)
     os.environ[
         CONFIG_FILE_PATH_ENV_STR

tests/mxnet/test_json_configs/checkpointconfig.json

@@ -1,3 +1,3 @@
 {
-  "LocalPath" : "./savedParams"
+  "LocalPath" : "/tmp/savedParams"
 }

tests/mxnet/test_json_configs/test_hook_all_zero_hook.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "newlogsRunTest2/test_hook_all_zero_hook_from_json",
+  "LocalPath": "/tmp/newlogsRunTest2/test_hook_all_zero_hook_from_json",
   "HookParameters": {
     "save_all": false,
     "save_steps": "0,1,2,3"

tests/mxnet/test_json_configs/test_hook_from_json_config.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "newlogsRunTest1/test_hook_from_json_config",
+  "LocalPath": "/tmp/newlogsRunTest1/test_hook_from_json_config",
   "HookParameters": {
     "save_steps": "0,1,2,3"
   }

tests/mxnet/test_json_configs/test_hook_from_json_config_full.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "newlogsRunTest2/test_hook_from_json_config_full",
+  "LocalPath": "/tmp/newlogsRunTest2/test_hook_from_json_config_full",
   "HookParameters": {
     "save_all": false,
     "include_regex": "regexe1,regex2",

tests/mxnet/test_json_configs/test_hook_reduce_config_hook.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "newlogsRunTest2/test_hook_reduce_config_hook_from_json",
+  "LocalPath": "/tmp/newlogsRunTest2/test_hook_reduce_config_hook_from_json",
   "HookParameters": {
     "save_all": false,
     "reductions": "max,mean",

tests/mxnet/test_json_configs/test_hook_save_all_hook.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "newlogsRunTest2/test_hook_save_all_hook_from_json",
+  "LocalPath": "/tmp/newlogsRunTest2/test_hook_save_all_hook_from_json",
   "HookParameters": {
     "save_all": true,
     "save_steps": "0,1,2,3"

tests/mxnet/test_json_configs/test_modes_hook.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "newlogsRunTest2/test_modes_hookjson",
+  "LocalPath": "/tmp/test_modes_hookjson",
   "HookParameters": {
     "train.save_interval": 2,
     "eval.save_interval": 3

tests/mxnet/test_json_configs/test_save_config_hookjson_config.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "newlogsRunTest2/test_hook_from_json_config_full",
+  "LocalPath": "/tmp/test_hook_from_json_config_full",
   "HookParameters": {
     "save_all": false,
     "save_steps": "0,1,2,3"

tests/mxnet/test_modes.py

@@ -14,7 +14,7 @@
 def test_modes(hook=None, path=None):
     if hook is None:
         run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-        path = "./newlogsRunTest/" + run_id
+        path = "/tmp/" + run_id
         hook = t_hook(
             out_dir=path,
             save_config=SaveConfig(
@@ -57,7 +57,7 @@ def test_modes_hook_from_json_config():
     import shutil
     import os
 
-    out_dir = "newlogsRunTest2/test_modes_hookjson"
+    out_dir = "/tmp/test_modes_hookjson"
     shutil.rmtree(out_dir, True)
     os.environ[CONFIG_FILE_PATH_ENV_STR] = "tests/mxnet/test_json_configs/test_modes_hook.json"
     hook = t_hook.create_from_json_file()

tests/mxnet/test_spot_training.py

@@ -143,7 +143,7 @@ def test_spot_hook():
     os.environ[
         CHECKPOINT_CONFIG_FILE_PATH_ENV_VAR
     ] = "./tests/mxnet/test_json_configs/checkpointconfig.json"
-    checkpoint_path = "./savedParams"
+    checkpoint_path = "/tmp/savedParams"
     if not os.path.exists(checkpoint_path):
         os.mkdir(checkpoint_path)
     save_config = SaveConfig(save_steps=[10, 11, 12, 13, 14, 40, 50, 60, 70, 80])
@@ -154,7 +154,7 @@ def test_spot_hook():
     """
 
     run_id_1 = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-    out_dir_1 = "newlogsRunTest/" + run_id_1
+    out_dir_1 = "/tmp/newlogsRunTest/" + run_id_1
     hook = t_hook(
         out_dir=out_dir_1, save_config=save_config, include_collections=["weights", "gradients"]
     )
@@ -174,7 +174,7 @@ def test_spot_hook():
     We expect to read steps 40, 50, 60, 70 and 80
     """
     run_id_2 = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-    out_dir_2 = "newlogsRunTest/" + run_id_2
+    out_dir_2 = "/tmp/newlogsRunTest/" + run_id_2
     hook = t_hook(
         out_dir=out_dir_2, save_config=save_config, include_collections=["weights", "gradients"]
     )
@@ -216,4 +216,4 @@ def test_spot_hook():
 
     print("Cleaning up.")
     shutil.rmtree(os.path.dirname(out_dir_1))
-    shutil.rmtree(checkpoint_path)
+    shutil.rmtree(checkpoint_path, ignore_errors=True)

tests/mxnet/test_training_end.py

@@ -15,7 +15,7 @@
 @pytest.mark.slow  # 0:03 to run
 def test_end_local_training():
     run_id = str(uuid.uuid4())
-    out_dir = "./newlogsRunTest/" + run_id
+    out_dir = "/tmp/newlogsRunTest/" + run_id
     assert has_training_ended(out_dir) == False
     subprocess.check_call(
         [

tests/pytorch/test_collection.py

@@ -19,7 +19,7 @@ def test_collection_add(hook=None, out_dir=None):
     hook_created = False
     if hook is None:
         run_id = "trial_" + datetime.now().strftime("%Y%m%d-%H%M%S%f")
-        out_dir = "./newlogsRunTest/" + run_id
+        out_dir = "/tmp/" + run_id
         hook = t_hook(
             out_dir=out_dir,
             save_config=SaveConfig(save_steps=[0, 1, 2, 3]),

tests/pytorch/test_json_configs/test_hook_multi_collections.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "test_output/test_hook_multi_collection/jsonloading",
+  "LocalPath": "/tmp/test_output/test_hook_multi_collection/jsonloading",
   "HookParameters": {
     "save_all": false,
     "save_steps": "0, 1, 2, 3"

tests/pytorch/test_json_configs/test_hook_reduction_config.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "test_output/test_hook_reduction_config/jsonloading",
+  "LocalPath": "/tmp/test_output/test_hook_reduction_config/jsonloading",
   "HookParameters": {
     "save_all": false,
     "reductions": "max, mean, variance",

tests/pytorch/test_json_configs/test_hook_saveall.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "test_output/test_hook_saveall/jsonloading",
+  "LocalPath": "/tmp/test_output/test_hook_saveall/jsonloading",
   "HookParameters": {
     "save_all": true,
     "save_steps": "0,20,40,60,80"

tests/pytorch/test_json_configs/test_hook_weightsbiasgradients.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "test_output/test_hook_save_weightsbiasgradients/jsonloading",
+  "LocalPath": "/tmp/test_output/test_hook_save_weightsbiasgradients/jsonloading",
   "HookParameters": {
     "save_all": false,
     "save_steps": "0,20,40,60,80"

tests/pytorch/test_json_configs/test_modes.json

@@ -1,6 +1,6 @@
 {
   "S3Path": "s3://kjndjknd_bucket/prefix",
-  "LocalPath": "test_output/test_hook_modes/jsonloading",
+  "LocalPath": "/tmp/test_output/test_hook_modes/jsonloading",
   "HookParameters": {
     "save_all": true,
     "train.save_steps": "0,1,2,3,4"