Merge pull request #13 from andrewztan/tune_cli

Tune cli to be merged to ray

doc/source/tune-usage.rst

@@ -499,6 +499,49 @@ And stopping a trial (``PUT /trials/[:id]``):
     curl -X PUT http://<address>:<port>/trials/<trial_id>
 
 
+Tune CLI (Experimental)
+-----------------------
+
+``tune`` is a powerful, easy-to-use command line interface (CLI) to manage and monitor your experiments on Ray. To do this, verify that you have the ``tabulate`` library installed:
+
+.. code-block:: bash
+
+    $ pip install tabulate
+
+Here are a few examples of command line calls that can be made and the corresponding terminal output. The commands will resize your output based on the size of your terminal.
+
+- ``tune ls``: List tabular information about trials within an experiment. Add the ``--sort`` flag to sort the output by specific columns. Empty columns will be removed and reported by default. If the terminal window is not wide enough to fit the entire table, columns will be dropped and reported based on the order of fields requested.
+
+.. code-block:: bash
+
+    $ tune ls path_to_experiment --sort trial_id
+    +----+------------+------------+----------------+
+    |    | trial_id   | status     |   num_failures |
+    |----+------------+------------+----------------|
+    |  0 | 2a51423e   | TERMINATED |              0 |
+    |  1 | 930a2a93   | TERMINATED |              0 |
+    |  2 | a3be6d03   | TERMINATED |              0 |
+    |  3 | c86c6d43   | TERMINATED |              0 |
+    +----+------------+------------+----------------+
+    Dropped columns: ['logdir']
+    Please increase your terminal size to view remaining columns.
+    Empty columns: ['trial_name']
+
+- ``tune lsx``: List tabular information about experiments within a project. Add the ``--sort`` flag to sort the output by specific columns. Add the ``--sort`` flag to sort the output by specific columns. Empty columns will be removed and reported by default. If the terminal window is not wide enough to fit the entire table, columns will be dropped and reported based on the order of fields requested.
+
+.. code-block:: bash
+
+    $ tune lsx path_to_project
+    +----+--------+--------------+----------------+-------------------+--------------+
+    |    | name   | total_trials | running_trials | terminated_trials | error_trials |
+    |----+--------+--------------+----------------+-------------------+--------------|
+    |  0 | exp_1  |            1 |              0 |                 0 |            1 |
+    |  1 | exp_2  |            1 |              0 |                 1 |            0 |
+    |  2 | exp_3  |            6 |              0 |                 6 |            0 |
+    +----+--------+--------------+----------------+-------------------+--------------+
+    Empty columns: ['timestamp']
+
+
 Further Questions or Issues?
 ----------------------------
 

docker/examples/Dockerfile

@@ -12,4 +12,5 @@ RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git
 RUN pip install --upgrade sigopt
 # RUN pip install --upgrade nevergrad
 RUN pip install --upgrade scikit-optimize
+RUN pip install --upgrade tabulate
 RUN conda install pytorch-cpu torchvision-cpu -c pytorch

python/ray/tune/scripts.py

@@ -6,6 +6,7 @@
 import glob
 import json
 import os
+import subprocess
 from datetime import datetime
 
 import pandas as pd
@@ -45,6 +46,10 @@ def cli():
     "error_trials",
 )
 
+TERMINAL_HEIGHT, TERMINAL_WIDTH = subprocess.check_output(['stty',
+                                                           'size']).split()
+TERMINAL_HEIGHT, TERMINAL_WIDTH = int(TERMINAL_HEIGHT), int(TERMINAL_WIDTH)
+
 
 def _list_trials(experiment_path, sort,
                  info_keys=DEFAULT_EXPERIMENT_INFO_KEYS):
@@ -58,18 +63,36 @@ def _list_trials(experiment_path, sort,
     checkpoints_df = pd.DataFrame(
         experiment_state["checkpoints"])[list(info_keys)]
     if "logdir" in checkpoints_df.columns:
+        # logdir often too verbose to view in table, so drop experiment_path
         checkpoints_df["logdir"] = checkpoints_df["logdir"].str.replace(
             experiment_path, '')
     if sort:
         checkpoints_df = checkpoints_df.sort_values(by=sort)
-    print(tabulate(checkpoints_df, headers="keys", tablefmt="psql"))
-
-    # TODO(hartikainen): The logdir is often too verbose to be viewed in a
-    # table.
-    # checkpoints = pd.DataFrame.from_records(experiment_state['checkpoints'])
-    # checkpoints['logdir'] = checkpoints['logdir'].str.replace(
-    #     experiment_path, '')
-    # print(checkpoints[list(info_keys)].to_string())
+        checkpoints_df = checkpoints_df.reset_index(drop=True)
+
+    print_df = pd.DataFrame()
+    columns = list(info_keys)
+    dropped = []
+    empty = []
+    # column display priority is based on the info_keys passed in
+    for i in range(len(columns)):
+        print_df[columns[i]] = checkpoints_df[columns[i]]
+        table = tabulate(print_df, headers="keys", tablefmt="psql")
+        if checkpoints_df[columns[i]].isnull().all():
+            empty.append(columns[i])
+            print_df.drop(columns[i], axis=1, inplace=True)
+        elif str(table).index('\n') > TERMINAL_WIDTH:
+            print_df.drop(columns[i], axis=1, inplace=True)
+            table = tabulate(print_df, headers="keys", tablefmt="psql")
+            dropped += columns[i:]
+            break
+
+    print(table)
+    if dropped:
+        print("Dropped columns:", dropped)
+        print("Please increase your terminal size to view remaining columns.")
+    if empty:
+        print("Empty columns:", empty)
 
 
 @cli.command()
@@ -110,13 +133,36 @@ def _list_experiments(project_path, sort, info_keys=DEFAULT_PROJECT_INFO_KEYS):
                 checkpoints["status"] == Trial.TERMINATED).sum(),
             "error_trials": (checkpoints["status"] == Trial.ERROR).sum(),
         }
-
         experiment_data_collection.append(experiment_data)
 
-    info_df = pd.DataFrame(experiment_data_collection)
+    info_df = pd.DataFrame(experiment_data_collection)[list(info_keys)]
     if sort:
         info_df = info_df.sort_values(by=sort)
-    print(tabulate(info_df, headers="keys", tablefmt="psql"))
+        info_df = info_df.reset_index(drop=True)
+
+    print_df = pd.DataFrame()
+    columns = list(info_keys)
+    dropped = []
+    empty = []
+    # column display priority is based on the info_keys passed in
+    for i in range(len(columns)):
+        print_df[columns[i]] = info_df[columns[i]]
+        table = tabulate(print_df, headers="keys", tablefmt="psql")
+        if info_df[columns[i]].isnull().all():
+            empty.append(columns[i])
+            print_df.drop(columns[i], axis=1, inplace=True)
+        elif str(table).index('\n') > TERMINAL_WIDTH:
+            print_df.drop(columns[i], axis=1, inplace=True)
+            table = tabulate(print_df, headers="keys", tablefmt="psql")
+            dropped += columns[i:]
+            break
+
+    print(table)
+    if dropped:
+        print("Dropped columns:", dropped)
+        print("Please increase your terminal size to view remaining columns.")
+    if empty:
+        print("Empty columns:", empty)
 
 
 @cli.command()