chore: migrate experiment list api [DET-3696, DET-3697, DET-3698] (#1228

)

docs/release-notes/1228-migrate-experiments-list-api.txt

@@ -0,0 +1,9 @@
+:orphan:
+
+**BUG FIXES**
+
+- Migrate to new experiments list endpoint to support table pagination.
+
+**NEW FEATURES**
+
+- Add number of trials per experiment as a column in experiments list page.

proto/src/determined/experiment/v1/experiment.proto

@@ -4,6 +4,7 @@ package determined.experiment.v1;
 option go_package = "github.com/determined-ai/determined/proto/pkg/experimentv1";
 
 import "google/protobuf/timestamp.proto";
+import "protoc-gen-swagger/options/annotations.proto";
 
 // The current state of the experiment.
 enum State {
@@ -32,6 +33,11 @@ enum State {
 // Experiment is a collection of one or more trials that are exploring a user-defined
 // hyperparameter space.
 message Experiment {
+    option (grpc.gateway.protoc_gen_swagger.options.openapiv2_schema) = {
+        json_schema: {
+            required: ["id", "description", "start_time", "end_time", "state", "archived", "num_trials", "username"]
+        }
+    };
     // The id of the experiment.
     int32 id = 1;
     // The description of the experiment.

webui/react/src/App.tsx

@@ -12,7 +12,6 @@ import AppContexts from 'contexts/AppContexts';
 import Auth from 'contexts/Auth';
 import ClusterOverview from 'contexts/ClusterOverview';
 import { Commands, Notebooks, Shells, Tensorboards } from 'contexts/Commands';
-import Experiments from 'contexts/Experiments';
 import Info from 'contexts/Info';
 import UI from 'contexts/UI';
 import Users from 'contexts/Users';
@@ -115,7 +114,6 @@ const App: React.FC = () => {
       Agents.Provider,
       ClusterOverview.Provider,
       ActiveExperiments.Provider,
-      Experiments.Provider,
       Commands.Provider,
       Notebooks.Provider,
       Shells.Provider,

webui/react/src/components/Table.stories.tsx

@@ -3,7 +3,6 @@ import React from 'react';
 
 import { columns as experimentColumns } from 'pages/ExperimentList.table';
 import { columns as taskColumns } from 'pages/TaskList.table';
-import { ExperimentsDecorator } from 'storybook/ContextDecorators';
 import RouterDecorator from 'storybook/RouterDecorator';
 import { CommandTask, ExperimentItem } from 'types';
 import { generateCommandTask, generateExperiments } from 'utils/task';
@@ -13,14 +12,14 @@ import css from './Table.module.scss';
 
 export default {
   component: Table,
-  decorators: [ RouterDecorator, ExperimentsDecorator ],
+  decorators: [ RouterDecorator ],
   title: 'Table',
 };
 
 const commandTasks: CommandTask[] = new Array(20)
   .fill(null)
   .map((_, index) => generateCommandTask(index));
-const experimentItems: ExperimentItem[] = generateExperiments(30);
+const experiments: ExperimentItem[] = generateExperiments(30);
 
 export const LoadingTable = (): React.ReactNode => {
   return <Table loading={true} />;
@@ -45,8 +44,8 @@ export const ExperimentTable = (): React.ReactNode => {
   return <Table
     className={css.base}
     columns={experimentColumns}
-    dataSource={experimentItems}
-    loading={experimentItems === undefined}
+    dataSource={experiments}
+    loading={experiments === undefined}
     rowClassName={defaultRowClassName()}
     rowKey="id"
     showSorterTooltip={false} />;

webui/react/src/components/Table.tsx

@@ -6,26 +6,24 @@ import Avatar from 'components/Avatar';
 import Badge, { BadgeType } from 'components/Badge';
 import Icon from 'components/Icon';
 import ProgressBar from 'components/ProgressBar';
-import TaskActionDropdown from 'components/TaskActionDropdown';
 import {
-  CommandState, CommandTask, CommandType, ExperimentItem, RunState, StartEndTimes, TrialItem,
+  CommandState, CommandTask, CommandType, ExperimentItem,
+  Pagination, RunState, StartEndTimes, TrialItem,
 } from 'types';
 import { getDuration, shortEnglishHumannizer } from 'utils/time';
-import { commandTypeToLabel, experimentToTask } from 'utils/types';
+import { commandTypeToLabel } from 'utils/types';
 
 import css from './Table.module.scss';
 
 type TableRecord = CommandTask | ExperimentItem | TrialItem;
 
-export interface TableSorter {
-  descend: boolean;
-  key: string;
-}
-
-export interface TablePagination {
+export interface TablePaginationConfig {
+  current: number;
+  pageSize: number;
   defaultPageSize: number;
   hideOnSinglePage: boolean;
   showSizeChanger: boolean;
+  total: number;
 }
 
 export type Renderer<T = unknown> = (text: string, record: T, index: number) => React.ReactNode;
@@ -34,11 +32,23 @@ export type GenericRenderer = <T extends TableRecord>(
   text: string, record: T, index: number,
 ) => React.ReactNode;
 
-type ExperimentRenderer = (text: string, record: ExperimentItem, index: number) => React.ReactNode;
+export type ExperimentRenderer = (
+  text: string,
+  record: ExperimentItem,
+  index: number,
+) => React.ReactNode;
+
 export type TaskRenderer = (text: string, record: CommandTask, index: number) => React.ReactNode;
 
 export const MINIMUM_PAGE_SIZE = 10;
 
+export const defaultPaginationConfig = {
+  current: 1,
+  defaultPageSize: MINIMUM_PAGE_SIZE,
+  pageSize: MINIMUM_PAGE_SIZE,
+  showSizeChanger: true,
+};
+
 /* Table Column Renderers */
 
 export const durationRenderer = (times: StartEndTimes): React.ReactNode => {
@@ -69,8 +79,6 @@ export const userRenderer: Renderer<{ username: string }> = (_, record) => (
 
 /* Command Task Table Column Renderers */
 
-export const taskActionRenderer: TaskRenderer = (_, record) => <TaskActionDropdown task={record} />;
-
 export const taskIdRenderer: TaskRenderer = id => (
   <Tooltip placement="topLeft" title={id}>
     <div className={css.centerVertically}>
@@ -89,10 +97,6 @@ export const taskTypeRenderer: TaskRenderer = (_, record) => (
 
 /* Experiment Table Column Renderers */
 
-export const experimentActionRenderer: ExperimentRenderer = (_, record) => (
-  <TaskActionDropdown task={experimentToTask(record)} />
-);
-
 export const experimentDescriptionRenderer: ExperimentRenderer = (_, record) => {
   // TODO handle displaying labels not fitting the column width
   const labels = [ 'object detection', 'pytorch' ]; // TODO get from config
@@ -141,10 +145,24 @@ export const defaultRowClassName = (clickable = true): string=> {
   return clickable ? 'clickable' : '';
 };
 
-export const getPaginationConfig = (count: number): TablePagination => {
+export const getPaginationConfig = (count: number): Partial<TablePaginationConfig> => {
   return {
     defaultPageSize: MINIMUM_PAGE_SIZE,
     hideOnSinglePage: count < MINIMUM_PAGE_SIZE,
     showSizeChanger: true,
   };
 };
+
+export const getFullPaginationConfig = (
+  pagination: Pagination,
+  total: number,
+): TablePaginationConfig => {
+  return {
+    current: Math.floor(pagination.offset / pagination.limit) + 1,
+    defaultPageSize: MINIMUM_PAGE_SIZE,
+    hideOnSinglePage: total < MINIMUM_PAGE_SIZE,
+    pageSize: pagination.limit,
+    showSizeChanger: true,
+    total,
+  };
+};

webui/react/src/components/TaskActionDropdown.stories.tsx

@@ -1,14 +1,12 @@
 import React from 'react';
 
-import { ExperimentsDecorator } from 'storybook/ContextDecorators';
 import { CommandState, CommandType, RunState } from 'types';
 import { generateCommandTask, generateExperimentTask } from 'utils/task';
 
 import TaskActionDropdown from './TaskActionDropdown';
 
 export default {
   component: TaskActionDropdown,
-  decorators: [ ExperimentsDecorator ],
   title: 'TaskActionDropdown',
 };
 

webui/react/src/components/TaskActionDropdown.tsx

@@ -1,12 +1,13 @@
+import { isNumber } from 'util';
+
 import { Dropdown, Menu } from 'antd';
 import { MenuInfo } from 'rc-menu/lib/interface';
 import React from 'react';
 
 import Icon from 'components/Icon';
-import Experiments from 'contexts/Experiments';
 import handleError, { ErrorLevel, ErrorType } from 'ErrorHandler';
 import { archiveExperiment, createTensorboard, killTask, setExperimentState } from 'services/api';
-import { AnyTask, CommandTask, Experiment, ExperimentTask, RunState, TBSourceType } from 'types';
+import { AnyTask, CommandTask, ExperimentTask, RunState, TBSourceType } from 'types';
 import { openBlank, openCommand } from 'utils/routes';
 import { capitalize } from 'utils/string';
 import { isExperimentTask } from 'utils/task';
@@ -16,11 +17,13 @@ import css from './TaskActionDropdown.module.scss';
 
 interface Props {
   task: AnyTask;
+  onComplete?: () => void;
 }
 
 const stopPropagation = (e: React.MouseEvent): void => e.stopPropagation();
 
-const TaskActionDropdown: React.FC<Props> = ({ task }: Props) => {
+const TaskActionDropdown: React.FC<Props> = ({ task, onComplete }: Props) => {
+  const id = isNumber(task.id) ? task.id : parseInt(task.id);
   const isExperiment = isExperimentTask(task);
   const isExperimentTerminal = terminalRunStates.has(task.state as RunState);
   const isArchivable = isExperiment && isExperimentTerminal && !(task as ExperimentTask).archived;
@@ -33,66 +36,47 @@ const TaskActionDropdown: React.FC<Props> = ({ task }: Props) => {
   const isCancelable = isExperiment
     && cancellableRunStates.includes(task.state as RunState);
 
-  const experimentsResponse = Experiments.useStateContext();
-  const setExperiments = Experiments.useActionContext();
-
-  // update the local state of a single experiment.
-  // TODO refactor to send change event back to parent via callback
-  const updateExperimentLocally = (updater: (arg0: Experiment) => Experiment): void => {
-    if (experimentsResponse.data) {
-      const experiments = experimentsResponse.data
-        .map(exp => exp.id.toString() === task.id ? updater(exp) : exp);
-      setExperiments({
-        type: Experiments.ActionType.Set,
-        value: { ...experimentsResponse, data: experiments },
-      });
-    }
-  };
-
   const handleMenuClick = async (params: MenuInfo): Promise<void> => {
     params.domEvent.stopPropagation();
     try {
       switch (params.key) { // Cases should match menu items.
         case 'activate':
           await setExperimentState({
-            experimentId: parseInt(task.id),
+            experimentId: id,
             state: RunState.Active,
           });
-          updateExperimentLocally(exp => ({ ...exp, state: RunState.Active }));
+          if (onComplete) onComplete();
           break;
         case 'archive':
           if (!isExperimentTask(task)) break;
-          await archiveExperiment(parseInt(task.id));
-          updateExperimentLocally(exp => ({ ...exp, archived: true }));
+          await archiveExperiment(id);
+          if (onComplete) onComplete();
           break;
         case 'cancel':
           await setExperimentState({
-            experimentId: parseInt(task.id),
+            experimentId: id,
             state: RunState.StoppingCanceled,
           });
-          updateExperimentLocally(exp => ({ ...exp, state: RunState.StoppingCanceled }));
+          if (onComplete) onComplete();
           break;
         case 'createTensorboard': {
           const tensorboard = await createTensorboard({
-            ids: [ parseInt(task.id) ],
+            ids: [ id ],
             type: TBSourceType.Experiment,
           });
           openCommand(tensorboard);
           break;
         }
         case 'kill':
           await killTask(task);
-          if (isExperiment) {
-            // We don't provide immediate updates for command types yet.
-            updateExperimentLocally(exp => ({ ...exp, state: RunState.StoppingCanceled }));
-          }
+          if (isExperiment && onComplete) onComplete();
           break;
         case 'pause':
           await setExperimentState({
-            experimentId: parseInt(task.id),
+            experimentId: id,
             state: RunState.Paused,
           });
-          updateExperimentLocally(exp => ({ ...exp, state: RunState.Paused }));
+          if (onComplete) onComplete();
           break;
         case 'viewLogs': {
           const taskType = (task as CommandTask).type.toLocaleLowerCase();
@@ -102,8 +86,8 @@ const TaskActionDropdown: React.FC<Props> = ({ task }: Props) => {
         }
         case 'unarchive':
           if (!isExperimentTask(task)) break;
-          await archiveExperiment(parseInt(task.id), false);
-          updateExperimentLocally(exp => ({ ...exp, archived: false }));
+          await archiveExperiment(id, false);
+          if (onComplete) onComplete();
       }
     } catch (e) {
       handleError({

webui/react/src/components/TaskCard.stories.tsx

@@ -1,7 +1,6 @@
 import React from 'react';
 
 import Grid from 'components/Grid';
-import { ExperimentsDecorator } from 'storybook/ContextDecorators';
 import RouterDecorator from 'storybook/RouterDecorator';
 import { ShirtSize } from 'themes';
 import { generateCommandTask, generateExperimentTask, generateTasks } from 'utils/task';
@@ -10,7 +9,7 @@ import TaskCard from './TaskCard';
 
 export default {
   component: TaskCard,
-  decorators: [ RouterDecorator, ExperimentsDecorator ],
+  decorators: [ RouterDecorator ],
   title: 'TaskCard',
 };
 

webui/react/src/components/TaskCard.tsx

@@ -6,7 +6,7 @@ import Icon from 'components/Icon';
 import Link from 'components/Link';
 import ProgressBar from 'components/ProgressBar';
 import TaskActionDropdown from 'components/TaskActionDropdown';
-import { RecentTask } from 'types';
+import { RecentCommandTask, RecentTask } from 'types';
 import { percent } from 'utils/number';
 import { canBeOpened, isExperimentTask } from 'utils/task';
 
@@ -15,12 +15,13 @@ import css from './TaskCard.module.scss';
 const TaskCard: React.FC<RecentTask> = (props: RecentTask) => {
   let [ hasProgress, isComplete ] = [ false, false ];
   if (isExperimentTask(props)) {
-    hasProgress = props.progress != null;
+    hasProgress = !!props.progress;
     isComplete = props.progress === 1;
   }
   const classes = [ css.base ];
 
-  const iconName = isExperimentTask(props) ? 'experiment' : props.type.toLowerCase();
+  const iconName = isExperimentTask(props) ?
+    'experiment' : (props as RecentCommandTask).type.toLowerCase();
   if (canBeOpened(props) && props.url) classes.push(css.link);
 
   return (
@@ -45,7 +46,7 @@ const TaskCard: React.FC<RecentTask> = (props: RecentTask) => {
         </div>
         <div className={css.lower}>
           <div className={css.badges}>
-            <Badge type={BadgeType.Default}>{props.id.slice(0,4)}</Badge>
+            <Badge type={BadgeType.Default}>{`${props.id}`.slice(0,4)}</Badge>
             <Badge state={props.state} type={BadgeType.State} />
             {isExperimentTask(props) && hasProgress && !isComplete
                 && <div className={css.percent}>{`${percent(props.progress || 0)}%`}</div>}

webui/react/src/contexts/ActiveExperiments.tsx

@@ -1,8 +1,8 @@
 import { generateContext } from 'contexts';
 import { RestApiState } from 'hooks/useRestApi';
-import { Experiment } from 'types';
+import { ExperimentBase } from 'types';
 
-const contextProvider = generateContext<RestApiState<Experiment[]>>({
+const contextProvider = generateContext<RestApiState<ExperimentBase[]>>({
   initialState: {
     errorCount: 0,
     hasLoaded: false,