Add forecasting to raiwidgets and add a forecasting notebook

.github/workflows/CI-e2e-notebooks.yml

@@ -49,7 +49,12 @@ jobs:
         operatingSystem: [ubuntu-latest, windows-latest]
         pythonVersion: [3.8, 3.9, "3.10"]
         flights: ["", "dataBalanceExperience"]
-        notebookGroup: ["nb_group_1", "nb_group_2"]
+        notebookGroup: ["nb_group_1", "nb_group_2", "nb_group_3"]
+        exclude:
+          # nb_group_3 includes only forecasting which doesn't change
+          # with the data balance experience flight
+          - notebookGroup: "nb_group_3"
+            flights: "dataBalanceExperience"
 
     runs-on: ${{ matrix.operatingSystem }}
 
@@ -127,6 +132,12 @@ jobs:
           yarn e2e-widget -n "responsibleaidashboard-diabetes-decision-making" -f ${{ matrix.flights }}
           yarn e2e-widget -n "responsibleaidashboard-multiclass-dnn-model-debugging" -f ${{ matrix.flights }}
 
+      - if: ${{ matrix.notebookGroup == 'nb_group_3'}}
+        name: Run widget tests
+        shell: bash -l {0}
+        run: |
+          yarn e2e-widget -n "responsibleaidashboard-orange-juice-forecasting" -f ${{ matrix.flights }}
+
       - name: Upload e2e test screen shot
         if: always()
         uses: actions/upload-artifact@v3

apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxOrangeJuiceForecasting/whatIfForecasting.spec.ts

@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import {
+  describeWhatIfForecasting,
+  modelAssessmentDatasets
+} from "@responsible-ai/e2e";
+
+const datasetShape = modelAssessmentDatasets.OrangeJuiceForecasting;
+describeWhatIfForecasting(datasetShape, "OrangeJuiceForecasting");

libs/e2e/src/index.ts

@@ -9,6 +9,7 @@ export * from "./lib/describer/modelAssessment/errorAnalysis/describeErrorAnalys
 export * from "./lib/describer/modelAssessment/featureImportances/individualFeatureImportance/describeIndividualFeatureImportance";
 export * from "./lib/describer/modelAssessment/modelOverview/describeModelOverview";
 export * from "./lib/describer/modelAssessment/whatIfCounterfactuals/describeWhatIf";
+export * from "./lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecasting";
 export * from "./lib/describer/modelAssessment/datasets/modelAssessmentDatasets";
 export * from "./lib/describer/modelAssessment/IModelAssessmentData";
 export * from "./lib/describer/modelAssessment/visionDataExplorer/describeVisionDataExplorer";

libs/e2e/src/lib/describer/modelAssessment/Constants.ts

@@ -177,6 +177,22 @@ export enum Locators {
   AggregateBalanceMeasuresTable = "#aggregateBalanceMeasures .ms-DetailsList",
   AggregateBalanceMeasuresTableColumns = "#aggregateBalanceMeasures .ms-DetailsList-headerWrapper div[aria-label]",
   AggregateBalanceMeasuresTableRows = "#aggregateBalanceMeasures .ms-DetailsRow",
+  ForecastingDashboard = "#ModelAssessmentDashboard #ForecastingDashboard",
+  ForecastingTimeSeriesDropdown = "#ForecastingDashboard #ForecastingTimeSeriesDropdown",
+  ForecastingTimeSeriesDropdownOptions = "#ForecastingDashboard button[role='option']",
+  ForecastingTransformationCreationButton = "#ForecastingWhatIfTransformationCreationButton",
+  ForecastingTransformationsTable = "#ForecastingDashboard #ForecastingWhatIfTransformationsTable",
+  ForecastingTransformationValueField = "#ForecastingWhatIfTransformationValueField",
+  ForecastingTransformationNameField = "#ForecastingWhatIfTransformationNameField",
+  ForecastingTransformationAddButton = "#ForecastingWhatIfAddTransformationButton",
+  ForecastingTransformationFeatureDropdown = "#ForecastingWhatIfTransformationFeatureDropdown",
+  ForecastingTransformationFeatureDropdownOptions = "div.ms-ComboBox-optionsContainerWrapper button[role='option']",
+  ForecastingTransformationOperationDropdown = "#ForecastingWhatIfTransformationOperationDropdown",
+  ForecastingTransformationOperationDropdownWrapper = "#ForecastingWhatIfTransformationOperationDropdownwrapper",
+  ForecastingTransformationOperationDropdownOptions = "div.ms-ComboBox-optionsContainerWrapper button[role='option']",
+  ForecastingScenarioChart = "#ForecastingDashboard #ForecastScenarioChart",
+  ForecastingScenarioChartCurves = "#ForecastingDashboard .highcharts-series-group .highcharts-series",
+  ForecastingScenarioChartLegendItems = "#ForecastingDashboard .highcharts-a11y-proxy-button",
   VisionDataExplorer = "#VisionDataExplorer",
   VisionDataExplorerCohortDropDown = "#VisionDataExplorer #dataExplorerCohortDropdown",
   VisionDataExplorerSearchBox = "#VisionDataExplorer #dataExplorerSearchBox",

libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts

@@ -11,6 +11,7 @@ export interface IModelAssessmentData {
   dataBalanceData?: IDataBalanceData;
   causalAnalysisData?: ICausalAnalysisData;
   whatIfCounterfactualsData?: IWhatIfCounterfactualsData;
+  whatIfForecastingData?: IWhatIfForecastingData;
   featureNames?: string[];
   cohortDefaultName?: string;
   checkDupCohort?: boolean;
@@ -174,6 +175,18 @@ export interface IWhatIfCounterfactualsData {
   newClassValue?: string;
 }
 
+export interface IWhatIfForecastingData {
+  hasWhatIfForecastingComponent?: boolean;
+  numberOfTimeSeriesOptions?: number;
+  timeSeriesToSelect?: string;
+  testTransformation?: {
+    featureToSelect?: string;
+    operationToSelect?: string;
+    operationToSelectIndex?: number;
+    valueToSelect?: number;
+  };
+}
+
 export enum RAINotebookNames {
   "CensusClassificationModelDebugging" = "responsibleaidashboard-census-classification-model-debugging.py",
   "CensusClassificationModelDebuggingDataBalanceExperience" = "responsibleaidashboard-census-classification-model-debugging.py",
@@ -187,6 +200,8 @@ export enum RAINotebookNames {
   "HousingDecisionMakingDataBalanceExperience" = "responsibleaidashboard-housing-decision-making.py",
   "MulticlassDnnModelDebugging" = "responsibleaidashboard-multiclass-dnn-model-debugging.py",
   "MulticlassDnnModelDebuggingDataBalanceExperience" = "responsibleaidashboard-multiclass-dnn-model-debugging.py",
+  "OrangeJuiceForecasting" = "responsibleaidashboard-orange-juice-forecasting.py",
+  "OrangeJuiceForecastingDataBalanceExperience" = "responsibleaidashboard-orange-juice-forecasting.py",
   "FridgeImageClassificationModelDebugging" = "responsibleaidashboard-fridge-image-classification-model-debugging.py",
   "FridgeMultilabelModelDebugging" = "responsibleaidashboard-fridge-multilabel-image-classification-model-debugging.py",
   "FridgeObjectDetectionModelDebugging" = "responsibleaidashboard-fridge-object-detection-model-debugging.py"

libs/e2e/src/lib/describer/modelAssessment/datasets/OrangeJuiceForecasting.ts

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+export const OrangeJuiceForecasting = {
+  featureNames: [
+    "Advert",
+    "Price",
+    "Age60",
+    "COLLEGE",
+    "INCOME",
+    "Hincome150",
+    "Large HH",
+    "Minorities",
+    "WorkingWoman",
+    "SSTRDIST",
+    "SSTRVOL",
+    "CPDIST5",
+    "CPWVOL5"
+  ],
+  whatIfForecastingData: {
+    hasWhatIfForecastingComponent: true,
+    numberOfTimeSeriesOptions: 9,
+    testTransformation: {
+      featureToSelect: "INCOME",
+      operationToSelect: "multiply",
+      operationToSelectIndex: 0,
+      valueToSelect: 10
+    },
+    timeSeriesToSelect: "Store = 8, Brand = tropicana"
+  }
+};

libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts

@@ -15,6 +15,7 @@ import { HousingClassificationModelDebugging } from "./HousingClassificationMode
 import { HousingDecisionMaking } from "./HousingDecisionMaking";
 import { HousingRegression } from "./HousingRegression";
 import { MulticlassDnnModelDebugging } from "./MulticlassDnnModelDebugging";
+import { OrangeJuiceForecasting } from "./OrangeJuiceForecasting";
 
 export const regExForNumbersWithBrackets = /^\((\d+)\)$/; // Ex: (60)
 
@@ -28,7 +29,8 @@ const modelAssessmentDatasets: { [name: string]: IModelAssessmentData } = {
   HousingClassificationModelDebugging,
   HousingDecisionMaking,
   HousingRegression,
-  MulticlassDnnModelDebugging
+  MulticlassDnnModelDebugging,
+  OrangeJuiceForecasting
 };
 
 const modelAssessmentDatasetsDataBalanceExperience: {

libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecasting.ts

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { visit } from "../../../../util/visit";
+import { Locators } from "../Constants";
+import { modelAssessmentDatasets } from "../datasets/modelAssessmentDatasets";
+import { IModelAssessmentData } from "../IModelAssessmentData";
+
+import { describeWhatIfForecastingCreate } from "./describeWhatIfForecastingCreate";
+import { describeWhatIfForecastingCreateWhatIf } from "./describeWhatIfForecastingCreateWhatIf";
+
+const testName = "What If Forecasting";
+
+export function describeWhatIfForecasting(
+  datasetShape: IModelAssessmentData,
+  name?: keyof typeof modelAssessmentDatasets
+): void {
+  describe(testName, () => {
+    before(() => {
+      visit(name);
+      cy.get("#ModelAssessmentDashboard").should("exist");
+    });
+    if (!datasetShape.whatIfForecastingData?.hasWhatIfForecastingComponent) {
+      it("should not have 'What-If Forecasting' component for the notebook", () => {
+        cy.get(Locators.ForecastingDashboard).should("not.exist");
+      });
+    } else {
+      describeWhatIfForecastingCreate(datasetShape);
+      describeWhatIfForecastingCreateWhatIf(datasetShape);
+    }
+  });
+}

libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecastingCreate.ts

@@ -0,0 +1,32 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { selectDropdown } from "../../../../util/dropdown";
+import { Locators } from "../Constants";
+import { IModelAssessmentData } from "../IModelAssessmentData";
+
+export function describeWhatIfForecastingCreate(
+  dataShape: IModelAssessmentData
+): void {
+  describe("What if Forecasting Create", () => {
+    it("Should be able to select time series", () => {
+      cy.get(Locators.ForecastingScenarioChart).should("not.exist");
+      cy.get(Locators.ForecastingTimeSeriesDropdown).should("exist").click();
+      cy.get(Locators.ForecastingTimeSeriesDropdownOptions).should(
+        "have.length",
+        dataShape.whatIfForecastingData?.numberOfTimeSeriesOptions
+      );
+      // click again to close dropdown ahead of selectDropdown
+      cy.get(Locators.ForecastingTimeSeriesDropdown).click();
+      if (dataShape.whatIfForecastingData?.timeSeriesToSelect) {
+        selectDropdown(
+          Locators.ForecastingTimeSeriesDropdown,
+          dataShape.whatIfForecastingData?.timeSeriesToSelect
+        );
+      }
+
+      cy.get(Locators.ForecastingScenarioChart).should("exist");
+      cy.get(Locators.ForecastingScenarioChartCurves).should("have.length", 2);
+    });
+  });
+}

libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecastingCreateWhatIf.ts

@@ -0,0 +1,79 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { selectDropdown } from "../../../../util/dropdown";
+import { Locators } from "../Constants";
+import { IModelAssessmentData } from "../IModelAssessmentData";
+
+export function describeWhatIfForecastingCreateWhatIf(
+  dataShape: IModelAssessmentData
+): void {
+  describe("What if Forecasting Create What If", () => {
+    it("Should be able to create a what if scenario", () => {
+      cy.get(Locators.ForecastingTransformationNameField).should("not.exist");
+      cy.get(Locators.ForecastingTransformationCreationButton)
+        .should("exist")
+        .click();
+      cy.get(Locators.ForecastingTransformationNameField)
+        .should("exist")
+        .type("test");
+
+      cy.get(Locators.ForecastingTransformationFeatureDropdown)
+        .should("exist")
+        .click();
+      cy.get(Locators.ForecastingTransformationFeatureDropdownOptions).should(
+        "have.length",
+        dataShape.featureNames?.length
+      );
+      // click again to close dropdown ahead of selectComboBox
+      cy.get(Locators.ForecastingTransformationFeatureDropdown)
+        .should("exist")
+        .click();
+      if (
+        dataShape.whatIfForecastingData?.testTransformation?.featureToSelect
+      ) {
+        selectDropdown(
+          Locators.ForecastingTransformationFeatureDropdown,
+          dataShape.whatIfForecastingData?.testTransformation?.featureToSelect
+        );
+      }
+      //cy.get(Locators.ForecastingTransformationOperationDropdown).should("exist").click();
+      cy.get(Locators.ForecastingTransformationOperationDropdownWrapper)
+        .should("exist")
+        .click();
+      cy.get(Locators.ForecastingTransformationOperationDropdownOptions).should(
+        "have.length",
+        4
+      );
+      // click again to close dropdown ahead of selectComboBox
+      cy.get(Locators.ForecastingTransformationOperationDropdownWrapper)
+        .should("exist")
+        .click();
+      cy.get(
+        `${Locators.ForecastingTransformationOperationDropdown} button.ms-ComboBox-CaretDown-button`
+      )
+        .click()
+        .get(
+          `div.ms-ComboBox-optionsContainerWrapper button:eq(${dataShape.whatIfForecastingData?.testTransformation?.operationToSelectIndex})`
+        )
+        .click();
+
+      const transformationValue =
+        dataShape.whatIfForecastingData?.testTransformation?.valueToSelect?.toString();
+      if (transformationValue) {
+        cy.get(Locators.ForecastingTransformationValueField)
+          .should("exist")
+          .type(transformationValue);
+      }
+      cy.get(Locators.ForecastingTransformationAddButton)
+        .should("exist")
+        .click();
+      cy.get(Locators.ForecastingTransformationNameField).should("not.exist");
+
+      cy.get(Locators.ForecastingTransformationsTable).should("exist");
+
+      cy.get(Locators.ForecastingScenarioChart).should("exist");
+      cy.get(Locators.ForecastingScenarioChartCurves).should("have.length", 3);
+    });
+  });
+}

libs/forecasting/src/lib/ForecastingDashboard/Controls/TransformationCreation.tsx

@@ -90,6 +90,7 @@ export class TransformationCreation extends React.Component<
             }
           </Label>
           <ComboBox
+            id={"ForecastingWhatIfTransformationFeatureDropdown"}
             errorMessage={
               this.props.transformationFeature === undefined
                 ? localization.Forecasting.TransformationCreation

libs/forecasting/src/lib/ForecastingDashboard/Controls/TransformationCreationContinuous.tsx

@@ -55,6 +55,7 @@ export class TransformationCreationContinuous extends React.Component<ITransform
               }
             </Label>
             <ComboBox
+              id={"ForecastingWhatIfTransformationOperationDropdown"}
               errorMessage={
                 this.props.transformationOperation === undefined
                   ? localization.Forecasting.TransformationCreation
@@ -90,6 +91,7 @@ export class TransformationCreationContinuous extends React.Component<ITransform
                 }
               </Label>
               <TextField
+                id={"ForecastingWhatIfTransformationValueField"}
                 type="number"
                 step={this.transformationValueStep}
                 value={this.props.transformationValue.toString()}

libs/forecasting/src/lib/ForecastingDashboard/Controls/TransformationCreationDialog.tsx

@@ -119,6 +119,7 @@ export class TransformationCreationDialog extends React.Component<
         >
           <Stack.Item>
             <TextField
+              id={"ForecastingWhatIfTransformationNameField"}
               label={localization.Forecasting.TransformationCreation.nameLabel}
               placeholder={
                 localization.Forecasting.TransformationCreation
@@ -155,6 +156,7 @@ export class TransformationCreationDialog extends React.Component<
         </Stack>
         <DialogFooter>
           <PrimaryButton
+            id={"ForecastingWhatIfAddTransformationButton"}
             disabled={
               transformationCombinationErrorMessage !== undefined ||
               this.getTransformationNameErrorMessage(

libs/forecasting/src/lib/ForecastingDashboard/Controls/WhatIfSection.tsx

@@ -35,14 +35,15 @@ export class WhatIfSection extends React.Component<IWhatIfSectionProps> {
       <>
         <Stack.Item>
           <PrimaryButton
+            id={"ForecastingWhatIfTransformationCreationButton"}
             disabled={false}
             onClick={this.props.onClickWhatIfButton}
             text={localization.Forecasting.TransformationCreation.title}
           />
         </Stack.Item>
 
         {this.props.transformations.size > 0 && (
-          <Stack.Item>
+          <Stack.Item id="ForecastingWhatIfTransformationsTable">
             <TransformationsTable
               transformations={this.props.transformations}
               jointDataset={this.context.jointDataset}

libs/forecasting/src/lib/ForecastingDashboard/ForecastingDashboard.tsx

@@ -101,7 +101,7 @@ export class ForecastingDashboard extends React.Component<
         <Stack
           className={classNames.sectionStack}
           tokens={{ childrenGap: "24px", padding: "0 0 100px 0" }}
-          id="ForecastingDashboard"
+          id={"ForecastingDashboard"}
         >
           <Stack.Item className={classNames.topLevelDescriptionText}>
             <Text>{description}</Text>
@@ -116,6 +116,7 @@ export class ForecastingDashboard extends React.Component<
               </Text>
             ) : (
               <Dropdown
+                id={"ForecastingTimeSeriesDropdown"}
                 label={localization.Forecasting.timeSeries}
                 className={classNames.dropdown}
                 options={this.state.timeSeriesOptions}