diff --git a/.github/workflows/CI-e2e-notebooks.yml b/.github/workflows/CI-e2e-notebooks.yml index 7afe752398..fbcacd5b13 100644 --- a/.github/workflows/CI-e2e-notebooks.yml +++ b/.github/workflows/CI-e2e-notebooks.yml @@ -49,7 +49,12 @@ jobs: operatingSystem: [ubuntu-latest, windows-latest] pythonVersion: [3.8, 3.9, "3.10"] flights: ["", "dataBalanceExperience"] - notebookGroup: ["nb_group_1", "nb_group_2"] + notebookGroup: ["nb_group_1", "nb_group_2", "nb_group_3"] + exclude: + # nb_group_3 includes only forecasting which doesn't change + # with the data balance experience flight + - notebookGroup: "nb_group_3" + flights: "dataBalanceExperience" runs-on: ${{ matrix.operatingSystem }} @@ -127,6 +132,12 @@ jobs: yarn e2e-widget -n "responsibleaidashboard-diabetes-decision-making" -f ${{ matrix.flights }} yarn e2e-widget -n "responsibleaidashboard-multiclass-dnn-model-debugging" -f ${{ matrix.flights }} + - if: ${{ matrix.notebookGroup == 'nb_group_3'}} + name: Run widget tests + shell: bash -l {0} + run: | + yarn e2e-widget -n "responsibleaidashboard-orange-juice-forecasting" -f ${{ matrix.flights }} + - name: Upload e2e test screen shot if: always() uses: actions/upload-artifact@v3 diff --git a/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxOrangeJuiceForecasting/whatIfForecasting.spec.ts b/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxOrangeJuiceForecasting/whatIfForecasting.spec.ts new file mode 100644 index 0000000000..172faabd74 --- /dev/null +++ b/apps/widget-e2e/src/integration/modelAssessment/responsibleaitoolboxOrangeJuiceForecasting/whatIfForecasting.spec.ts @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + describeWhatIfForecasting, + modelAssessmentDatasets +} from "@responsible-ai/e2e"; + +const datasetShape = modelAssessmentDatasets.OrangeJuiceForecasting; +describeWhatIfForecasting(datasetShape, "OrangeJuiceForecasting"); diff --git a/libs/e2e/src/index.ts b/libs/e2e/src/index.ts index 53ea52ef88..2251237777 100644 --- a/libs/e2e/src/index.ts +++ b/libs/e2e/src/index.ts @@ -9,6 +9,7 @@ export * from "./lib/describer/modelAssessment/errorAnalysis/describeErrorAnalys export * from "./lib/describer/modelAssessment/featureImportances/individualFeatureImportance/describeIndividualFeatureImportance"; export * from "./lib/describer/modelAssessment/modelOverview/describeModelOverview"; export * from "./lib/describer/modelAssessment/whatIfCounterfactuals/describeWhatIf"; +export * from "./lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecasting"; export * from "./lib/describer/modelAssessment/datasets/modelAssessmentDatasets"; export * from "./lib/describer/modelAssessment/IModelAssessmentData"; export * from "./lib/describer/modelAssessment/visionDataExplorer/describeVisionDataExplorer"; diff --git a/libs/e2e/src/lib/describer/modelAssessment/Constants.ts b/libs/e2e/src/lib/describer/modelAssessment/Constants.ts index 1597d44caa..d0d45aaa91 100644 --- a/libs/e2e/src/lib/describer/modelAssessment/Constants.ts +++ b/libs/e2e/src/lib/describer/modelAssessment/Constants.ts @@ -177,6 +177,22 @@ export enum Locators { AggregateBalanceMeasuresTable = "#aggregateBalanceMeasures .ms-DetailsList", AggregateBalanceMeasuresTableColumns = "#aggregateBalanceMeasures .ms-DetailsList-headerWrapper div[aria-label]", AggregateBalanceMeasuresTableRows = "#aggregateBalanceMeasures .ms-DetailsRow", + ForecastingDashboard = "#ModelAssessmentDashboard #ForecastingDashboard", + ForecastingTimeSeriesDropdown = "#ForecastingDashboard #ForecastingTimeSeriesDropdown", + ForecastingTimeSeriesDropdownOptions = "#ForecastingDashboard button[role='option']", + ForecastingTransformationCreationButton = "#ForecastingWhatIfTransformationCreationButton", + ForecastingTransformationsTable = "#ForecastingDashboard #ForecastingWhatIfTransformationsTable", + ForecastingTransformationValueField = "#ForecastingWhatIfTransformationValueField", + ForecastingTransformationNameField = "#ForecastingWhatIfTransformationNameField", + ForecastingTransformationAddButton = "#ForecastingWhatIfAddTransformationButton", + ForecastingTransformationFeatureDropdown = "#ForecastingWhatIfTransformationFeatureDropdown", + ForecastingTransformationFeatureDropdownOptions = "div.ms-ComboBox-optionsContainerWrapper button[role='option']", + ForecastingTransformationOperationDropdown = "#ForecastingWhatIfTransformationOperationDropdown", + ForecastingTransformationOperationDropdownWrapper = "#ForecastingWhatIfTransformationOperationDropdownwrapper", + ForecastingTransformationOperationDropdownOptions = "div.ms-ComboBox-optionsContainerWrapper button[role='option']", + ForecastingScenarioChart = "#ForecastingDashboard #ForecastScenarioChart", + ForecastingScenarioChartCurves = "#ForecastingDashboard .highcharts-series-group .highcharts-series", + ForecastingScenarioChartLegendItems = "#ForecastingDashboard .highcharts-a11y-proxy-button", VisionDataExplorer = "#VisionDataExplorer", VisionDataExplorerCohortDropDown = "#VisionDataExplorer #dataExplorerCohortDropdown", VisionDataExplorerSearchBox = "#VisionDataExplorer #dataExplorerSearchBox", diff --git a/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts b/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts index d5016e2e0c..e21c493e6e 100644 --- a/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts +++ b/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts @@ -11,6 +11,7 @@ export interface IModelAssessmentData { dataBalanceData?: IDataBalanceData; causalAnalysisData?: ICausalAnalysisData; whatIfCounterfactualsData?: IWhatIfCounterfactualsData; + whatIfForecastingData?: IWhatIfForecastingData; featureNames?: string[]; cohortDefaultName?: string; checkDupCohort?: boolean; @@ -174,6 +175,18 @@ export interface IWhatIfCounterfactualsData { newClassValue?: string; } +export interface IWhatIfForecastingData { + hasWhatIfForecastingComponent?: boolean; + numberOfTimeSeriesOptions?: number; + timeSeriesToSelect?: string; + testTransformation?: { + featureToSelect?: string; + operationToSelect?: string; + operationToSelectIndex?: number; + valueToSelect?: number; + }; +} + export enum RAINotebookNames { "CensusClassificationModelDebugging" = "responsibleaidashboard-census-classification-model-debugging.py", "CensusClassificationModelDebuggingDataBalanceExperience" = "responsibleaidashboard-census-classification-model-debugging.py", @@ -187,6 +200,8 @@ export enum RAINotebookNames { "HousingDecisionMakingDataBalanceExperience" = "responsibleaidashboard-housing-decision-making.py", "MulticlassDnnModelDebugging" = "responsibleaidashboard-multiclass-dnn-model-debugging.py", "MulticlassDnnModelDebuggingDataBalanceExperience" = "responsibleaidashboard-multiclass-dnn-model-debugging.py", + "OrangeJuiceForecasting" = "responsibleaidashboard-orange-juice-forecasting.py", + "OrangeJuiceForecastingDataBalanceExperience" = "responsibleaidashboard-orange-juice-forecasting.py", "FridgeImageClassificationModelDebugging" = "responsibleaidashboard-fridge-image-classification-model-debugging.py", "FridgeMultilabelModelDebugging" = "responsibleaidashboard-fridge-multilabel-image-classification-model-debugging.py", "FridgeObjectDetectionModelDebugging" = "responsibleaidashboard-fridge-object-detection-model-debugging.py" diff --git a/libs/e2e/src/lib/describer/modelAssessment/datasets/OrangeJuiceForecasting.ts b/libs/e2e/src/lib/describer/modelAssessment/datasets/OrangeJuiceForecasting.ts new file mode 100644 index 0000000000..31138a8aaa --- /dev/null +++ b/libs/e2e/src/lib/describer/modelAssessment/datasets/OrangeJuiceForecasting.ts @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export const OrangeJuiceForecasting = { + featureNames: [ + "Advert", + "Price", + "Age60", + "COLLEGE", + "INCOME", + "Hincome150", + "Large HH", + "Minorities", + "WorkingWoman", + "SSTRDIST", + "SSTRVOL", + "CPDIST5", + "CPWVOL5" + ], + whatIfForecastingData: { + hasWhatIfForecastingComponent: true, + numberOfTimeSeriesOptions: 9, + testTransformation: { + featureToSelect: "INCOME", + operationToSelect: "multiply", + operationToSelectIndex: 0, + valueToSelect: 10 + }, + timeSeriesToSelect: "Store = 8, Brand = tropicana" + } +}; diff --git a/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts b/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts index daf5db3c33..b0171a7f51 100644 --- a/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts +++ b/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts @@ -15,6 +15,7 @@ import { HousingClassificationModelDebugging } from "./HousingClassificationMode import { HousingDecisionMaking } from "./HousingDecisionMaking"; import { HousingRegression } from "./HousingRegression"; import { MulticlassDnnModelDebugging } from "./MulticlassDnnModelDebugging"; +import { OrangeJuiceForecasting } from "./OrangeJuiceForecasting"; export const regExForNumbersWithBrackets = /^\((\d+)\)$/; // Ex: (60) @@ -28,7 +29,8 @@ const modelAssessmentDatasets: { [name: string]: IModelAssessmentData } = { HousingClassificationModelDebugging, HousingDecisionMaking, HousingRegression, - MulticlassDnnModelDebugging + MulticlassDnnModelDebugging, + OrangeJuiceForecasting }; const modelAssessmentDatasetsDataBalanceExperience: { diff --git a/libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecasting.ts b/libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecasting.ts new file mode 100644 index 0000000000..a7e56870e5 --- /dev/null +++ b/libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecasting.ts @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { visit } from "../../../../util/visit"; +import { Locators } from "../Constants"; +import { modelAssessmentDatasets } from "../datasets/modelAssessmentDatasets"; +import { IModelAssessmentData } from "../IModelAssessmentData"; + +import { describeWhatIfForecastingCreate } from "./describeWhatIfForecastingCreate"; +import { describeWhatIfForecastingCreateWhatIf } from "./describeWhatIfForecastingCreateWhatIf"; + +const testName = "What If Forecasting"; + +export function describeWhatIfForecasting( + datasetShape: IModelAssessmentData, + name?: keyof typeof modelAssessmentDatasets +): void { + describe(testName, () => { + before(() => { + visit(name); + cy.get("#ModelAssessmentDashboard").should("exist"); + }); + if (!datasetShape.whatIfForecastingData?.hasWhatIfForecastingComponent) { + it("should not have 'What-If Forecasting' component for the notebook", () => { + cy.get(Locators.ForecastingDashboard).should("not.exist"); + }); + } else { + describeWhatIfForecastingCreate(datasetShape); + describeWhatIfForecastingCreateWhatIf(datasetShape); + } + }); +} diff --git a/libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecastingCreate.ts b/libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecastingCreate.ts new file mode 100644 index 0000000000..1640fb972d --- /dev/null +++ b/libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecastingCreate.ts @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { selectDropdown } from "../../../../util/dropdown"; +import { Locators } from "../Constants"; +import { IModelAssessmentData } from "../IModelAssessmentData"; + +export function describeWhatIfForecastingCreate( + dataShape: IModelAssessmentData +): void { + describe("What if Forecasting Create", () => { + it("Should be able to select time series", () => { + cy.get(Locators.ForecastingScenarioChart).should("not.exist"); + cy.get(Locators.ForecastingTimeSeriesDropdown).should("exist").click(); + cy.get(Locators.ForecastingTimeSeriesDropdownOptions).should( + "have.length", + dataShape.whatIfForecastingData?.numberOfTimeSeriesOptions + ); + // click again to close dropdown ahead of selectDropdown + cy.get(Locators.ForecastingTimeSeriesDropdown).click(); + if (dataShape.whatIfForecastingData?.timeSeriesToSelect) { + selectDropdown( + Locators.ForecastingTimeSeriesDropdown, + dataShape.whatIfForecastingData?.timeSeriesToSelect + ); + } + + cy.get(Locators.ForecastingScenarioChart).should("exist"); + cy.get(Locators.ForecastingScenarioChartCurves).should("have.length", 2); + }); + }); +} diff --git a/libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecastingCreateWhatIf.ts b/libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecastingCreateWhatIf.ts new file mode 100644 index 0000000000..5db99390f4 --- /dev/null +++ b/libs/e2e/src/lib/describer/modelAssessment/whatIfForecasting/describeWhatIfForecastingCreateWhatIf.ts @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { selectDropdown } from "../../../../util/dropdown"; +import { Locators } from "../Constants"; +import { IModelAssessmentData } from "../IModelAssessmentData"; + +export function describeWhatIfForecastingCreateWhatIf( + dataShape: IModelAssessmentData +): void { + describe("What if Forecasting Create What If", () => { + it("Should be able to create a what if scenario", () => { + cy.get(Locators.ForecastingTransformationNameField).should("not.exist"); + cy.get(Locators.ForecastingTransformationCreationButton) + .should("exist") + .click(); + cy.get(Locators.ForecastingTransformationNameField) + .should("exist") + .type("test"); + + cy.get(Locators.ForecastingTransformationFeatureDropdown) + .should("exist") + .click(); + cy.get(Locators.ForecastingTransformationFeatureDropdownOptions).should( + "have.length", + dataShape.featureNames?.length + ); + // click again to close dropdown ahead of selectComboBox + cy.get(Locators.ForecastingTransformationFeatureDropdown) + .should("exist") + .click(); + if ( + dataShape.whatIfForecastingData?.testTransformation?.featureToSelect + ) { + selectDropdown( + Locators.ForecastingTransformationFeatureDropdown, + dataShape.whatIfForecastingData?.testTransformation?.featureToSelect + ); + } + cy.get(Locators.ForecastingTransformationOperationDropdownWrapper) + .should("exist") + .click(); + cy.get(Locators.ForecastingTransformationOperationDropdownOptions).should( + "have.length", + 4 + ); + // click again to close dropdown ahead of selectComboBox + cy.get(Locators.ForecastingTransformationOperationDropdownWrapper) + .should("exist") + .click(); + cy.get( + `${Locators.ForecastingTransformationOperationDropdown} button.ms-ComboBox-CaretDown-button` + ) + .click() + .get( + `div.ms-ComboBox-optionsContainerWrapper button:eq(${dataShape.whatIfForecastingData?.testTransformation?.operationToSelectIndex})` + ) + .click(); + + const transformationValue = + dataShape.whatIfForecastingData?.testTransformation?.valueToSelect?.toString(); + if (transformationValue) { + cy.get(Locators.ForecastingTransformationValueField) + .should("exist") + .type(transformationValue); + } + cy.get(Locators.ForecastingTransformationAddButton) + .should("exist") + .click(); + cy.get(Locators.ForecastingTransformationNameField).should("not.exist"); + + cy.get(Locators.ForecastingTransformationsTable).should("exist"); + + cy.get(Locators.ForecastingScenarioChart).should("exist"); + cy.get(Locators.ForecastingScenarioChartCurves).should("have.length", 3); + }); + }); +} diff --git a/libs/forecasting/src/lib/ForecastingDashboard/Controls/TransformationCreation.tsx b/libs/forecasting/src/lib/ForecastingDashboard/Controls/TransformationCreation.tsx index 49899936a5..eb0635fab2 100644 --- a/libs/forecasting/src/lib/ForecastingDashboard/Controls/TransformationCreation.tsx +++ b/libs/forecasting/src/lib/ForecastingDashboard/Controls/TransformationCreation.tsx @@ -90,6 +90,7 @@ export class TransformationCreation extends React.Component< } { <> { {this.props.transformations.size > 0 && ( - + {description} @@ -116,6 +116,7 @@ export class ForecastingDashboard extends React.Component< ) : ( \n", + "In this example, we use sktime to train and assess a time-series forecasting model for multiple time-series.\n", + "\n", + "The examples in the follow code samples use the University of Chicago's Dominick's Finer Foods dataset to forecast orange juice sales. Dominick's was a grocery chain in the Chicago metropolitan area." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1670990788014 + } + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from sktime.forecasting.arima import AutoARIMA\n", + "from sktime.forecasting.base import ForecastingHorizon\n", + "from sktime.forecasting.model_selection import temporal_train_test_split" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data\n", + "You are now ready to load the historical orange juice sales data. We will load the CSV file into a plain pandas DataFrame; the time column in the CSV is called _WeekStarting_, so it will be specially parsed into the datetime type.\n", + "Each row in the DataFrame holds a quantity of weekly sales for an OJ brand at a single store. The data also includes the sales price, a flag indicating if the OJ brand was advertised in the store that week, and some customer demographic information based on the store location. For historical reasons, the data also include the logarithm of the sales quantity. The Dominick's grocery data is commonly used to illustrate econometric modeling techniques where logarithms of quantities are generally preferred. \n", + "\n", + "The task is now to build a time-series model for the _Quantity_ column. It is important to note that this dataset is comprised of many individual time-series - one for each unique combination of _Store_ and _Brand_. To distinguish the individual time-series, we define the **time_series_id_features** the columns whose values determine the boundaries between time-series: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1670990899201 + } + }, + "outputs": [], + "source": [ + "time_column_name = \"WeekStarting\"\n", + "time_series_id_features = [\"Store\", \"Brand\"]\n", + "dataset_location = \"https://raw.githubusercontent.com/Azure/azureml-examples/2fe81643865e1f4591e7734bd1a729093cafb826/v1/python-sdk/tutorials/automl-with-azureml/forecasting-orange-juice-sales/dominicks_OJ.csv\"\n", + "data = pd.read_csv(dataset_location, parse_dates=[time_column_name])\n", + "\n", + "# Drop the columns 'logQuantity' as it is a leaky feature.\n", + "data.drop(\"logQuantity\", axis=1, inplace=True)\n", + "\n", + "# Set up multi index with time series ID columns and time column.\n", + "data.set_index(time_series_id_features + [time_column_name], inplace=True, drop=True)\n", + "data = data.groupby(time_series_id_features).apply(lambda group: group.loc[group.name].asfreq(\"W-THU\").interpolate())\n", + "data.sort_index(inplace=True, ascending=[True, True, True])\n", + "\n", + "data.head(10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1670990902872 + } + }, + "outputs": [], + "source": [ + "nseries = data.groupby(time_series_id_features).ngroups\n", + "print(\"Data contains {0} individual time-series.\".format(nseries))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For demonstration purposes, we extract sales time-series for just a few of the stores:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1670990905562 + } + }, + "outputs": [], + "source": [ + "use_stores = [2, 5, 8]\n", + "use_brands = ['tropicana', 'dominicks', 'minute.maid']\n", + "data_subset = data.loc[(use_stores, use_brands, slice(None)), :]\n", + "nseries = data_subset.groupby(time_series_id_features).ngroups\n", + "print(f\"Data subset contains {nseries} individual time-series.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data Splitting\n", + "We now split the data into a training and a testing set for later forecast evaluation. The test set will contain the final 20 weeks of observed sales for each time-series. The splits should be stratified by series, so we use a group-by statement on the time series identifier columns." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "gather": { + "logged": 1670990907583 + } + }, + "outputs": [], + "source": [ + "target_column_name = \"Quantity\"\n", + "\n", + "y = pd.DataFrame(data_subset[target_column_name])\n", + "X = data_subset.drop(columns=[target_column_name])\n", + "fh_dates = pd.DatetimeIndex(y.index.get_level_values(2).unique().sort_values().to_list()[-20:], freq='W-THU')\n", + "fh = ForecastingHorizon(fh_dates, is_relative=False)\n", + "y_train, y_test, X_train, X_test = \\\n", + " temporal_train_test_split(\n", + " y=y,\n", + " X=X,\n", + " test_size=20)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Train" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can now submit a new training run. Depending on the data and number of iterations this operation may take several minutes.\n", + "Information from each iteration will be printed to the console. Validation errors and current status will be shown when setting `show_output=True` and the execution will be synchronous." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# When using sktime directly we need to drop the time and time series ID columns.\n", + "model = AutoARIMA(suppress_warnings=True, error_action=\"ignore\")\n", + "model.fit(y=y_train, X=X_train, fh=fh)\n", + "model.predict(fh=fh, X=X_test).head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model.predict_quantiles(fh=fh, X=X_test, alpha=[0.025, 0.975]).head()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Responsible AI Dashboard" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from raiwidgets import ResponsibleAIDashboard\n", + "from responsibleai import RAIInsights, FeatureMetadata\n", + "\n", + "# merge X, y, and the time and time series ID features into a single DataFrame\n", + "train = X_train.join(y_train).join(X_train.index.to_frame(index=True))\n", + "test = X_test.join(y_test).join(X_test.index.to_frame(index=True))\n", + "train.reset_index(drop=True, inplace=True)\n", + "test.reset_index(drop=True, inplace=True)\n", + "\n", + "feature_metadata = FeatureMetadata(\n", + " time_series_id_features=time_series_id_features, \n", + " categorical_features=time_series_id_features,\n", + " datetime_features=[time_column_name])\n", + "insights = RAIInsights(\n", + " model=model,\n", + " train=train,\n", + " test=test,\n", + " task_type=\"forecasting\",\n", + " target_column=target_column_name,\n", + " feature_metadata=feature_metadata,\n", + " forecasting_enabled=True)\n", + "\n", + "ResponsibleAIDashboard(insights)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "authors": [ + { + "name": "jialiu" + } + ], + "categories": [ + "SDK v1", + "how-to-use-azureml", + "automated-machine-learning" + ], + "category": "tutorial", + "celltoolbar": "Raw Cell Format", + "compute": [ + "Remote" + ], + "datasets": [ + "Orange Juice Sales" + ], + "deployment": [ + "Azure Container Instance" + ], + "exclude_from_index": false, + "framework": [ + "Azure ML AutoML" + ], + "friendly_name": "Forecasting orange juice sales with deployment", + "index_order": 1, + "kernel_info": { + "name": "python38-azureml" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.17" + }, + "microsoft": { + "host": { + "AzureML": { + "notebookHasBeenCompleted": true + } + } + }, + "nteract": { + "version": "nteract-front-end@1.0.0" + }, + "tags": [ + "None" + ], + "task": "Forecasting", + "vscode": { + "interpreter": { + "hash": "6424d405450b15a93ca3015242fc1e51ac658b1b4015ae2fef5559269d9e1e0c" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/test_notebooks.py b/notebooks/test_notebooks.py index 52dc720385..0de1cfa5de 100644 --- a/notebooks/test_notebooks.py +++ b/notebooks/test_notebooks.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import os +import sys import nbformat as nbf import papermill as pm @@ -293,6 +294,18 @@ def test_responsibleaidashboard_fridge_object_detection_model_debugging(): assay_one_notebook(nb_path, nb_name, test_values) +# skip forecasting in Python 3.6 because of dependency incompatibilities +@pytest.mark.notebooks +@pytest.mark.skipif(sys.version_info < (3, 7), + reason="skip forecasting for Python 3.6") +def test_responsibleaidashboard_orange_juice_forecasting(): + nb_path = TABULAR + nb_name = "responsibleaidashboard-orange-juice-forecasting" + + test_values = {} + assay_one_notebook(nb_path, nb_name, test_values) + + @pytest.mark.notebooks def test_responsibleaidashboard_getting_started(): nb_path = TABULAR diff --git a/raiwidgets/raiwidgets/responsibleai_dashboard.py b/raiwidgets/raiwidgets/responsibleai_dashboard.py index 07f4322d3b..df1ff30436 100644 --- a/raiwidgets/raiwidgets/responsibleai_dashboard.py +++ b/raiwidgets/raiwidgets/responsibleai_dashboard.py @@ -5,6 +5,7 @@ from flask import jsonify, request +from raiutils.models import ModelTask from raiwidgets.dashboard import Dashboard from raiwidgets.responsibleai_dashboard_input import \ ResponsibleAIDashboardInput @@ -13,7 +14,6 @@ class ResponsibleAIDashboard(Dashboard): """The dashboard class, wraps the dashboard component. - :param analysis: An object that represents an RAIInsights. :type analysis: RAIInsights :param public_ip: Optional. If running on a remote vm, @@ -48,41 +48,54 @@ def predict(): return jsonify(self.input.on_predict(data)) self.add_url_rule(predict, '/predict', methods=["POST"]) - def tree(): - data = request.get_json(force=True) - return jsonify(self.input.debug_ml(data)) - self.add_url_rule(tree, '/tree', methods=["POST"]) - - def matrix(): - data = request.get_json(force=True) - return jsonify(self.input.matrix(data)) - self.add_url_rule(matrix, '/matrix', methods=["POST"]) - - def causal_whatif(): - data = request.get_json(force=True) - return jsonify(self.input.causal_whatif(data)) - self.add_url_rule(causal_whatif, '/causal_whatif', methods=["POST"]) - - def global_causal_effects(): - data = request.get_json(force=True) - return jsonify(self.input.get_global_causal_effects(data)) - self.add_url_rule(global_causal_effects, '/global_causal_effects', - methods=["POST"]) - - def global_causal_policy(): - data = request.get_json(force=True) - return jsonify(self.input.get_global_causal_policy(data)) - self.add_url_rule(global_causal_policy, '/global_causal_policy', - methods=["POST"]) - - def importances(): - return jsonify(self.input.importances()) - self.add_url_rule(importances, '/importances', methods=["POST"]) - - def get_exp(): - data = request.get_json(force=True) - return jsonify(self.input.get_exp(data)) - self.add_url_rule(get_exp, '/get_exp', methods=["POST"]) + if analysis.task_type == ModelTask.FORECASTING: + def forecast(): + data = request.get_json(force=True) + return jsonify(self.input.forecast(data)) + self.add_url_rule(forecast, '/forecast', methods=["POST"]) + else: + def tree(): + data = request.get_json(force=True) + return jsonify(self.input.debug_ml(data)) + self.add_url_rule(tree, '/tree', methods=["POST"]) + + def matrix(): + data = request.get_json(force=True) + return jsonify(self.input.matrix(data)) + self.add_url_rule(matrix, '/matrix', methods=["POST"]) + + def causal_whatif(): + data = request.get_json(force=True) + return jsonify(self.input.causal_whatif(data)) + self.add_url_rule( + causal_whatif, + '/causal_whatif', + methods=["POST"]) + + def global_causal_effects(): + data = request.get_json(force=True) + return jsonify(self.input.get_global_causal_effects(data)) + self.add_url_rule( + global_causal_effects, + '/global_causal_effects', + methods=["POST"]) + + def global_causal_policy(): + data = request.get_json(force=True) + return jsonify(self.input.get_global_causal_policy(data)) + self.add_url_rule( + global_causal_policy, + '/global_causal_policy', + methods=["POST"]) + + def importances(): + return jsonify(self.input.importances()) + self.add_url_rule(importances, '/importances', methods=["POST"]) + + def get_exp(): + data = request.get_json(force=True) + return jsonify(self.input.get_exp(data)) + self.add_url_rule(get_exp, '/get_exp', methods=["POST"]) def get_object_detection_metrics(): data = request.get_json(force=True) diff --git a/raiwidgets/raiwidgets/responsibleai_dashboard_input.py b/raiwidgets/raiwidgets/responsibleai_dashboard_input.py index 69a9a49f1a..0df2fdf3f2 100644 --- a/raiwidgets/raiwidgets/responsibleai_dashboard_input.py +++ b/raiwidgets/raiwidgets/responsibleai_dashboard_input.py @@ -7,11 +7,11 @@ import numpy as np import pandas as pd -from erroranalysis._internal.constants import ModelTask, display_name_to_metric -from raiutils.cohort import Cohort +from erroranalysis._internal.constants import display_name_to_metric +from raiutils.cohort import Cohort, CohortFilter, CohortFilterMethods from raiutils.data_processing import convert_to_list, serialize_json_safe from raiutils.exceptions import UserConfigValidationException -from raiutils.models import is_classifier +from raiutils.models import ModelTask, is_classifier from raiwidgets.constants import ErrorMessages from raiwidgets.error_handling import _format_exception from raiwidgets.interfaces import WidgetRequestResponseConstants @@ -41,20 +41,50 @@ def __init__( self.dashboard_input = analysis.get_data() self._validate_cohort_list(cohort_list) - if cohort_list is not None: - # Add cohort_list to dashboard_input - self.dashboard_input.cohortData = cohort_list - else: - self.dashboard_input.cohortData = [] self._feature_length = len(self.dashboard_input.dataset.feature_names) if hasattr(analysis, ManagerNames.ERROR_ANALYSIS): self._error_analyzer = analysis.error_analysis._analyzer + def _generate_time_series_cohorts(self): + """Generate time series cohorts based on time series ID columns.""" + cohort_list = [] + ts_id_cols = self._analysis._feature_metadata.time_series_id_features + all_time_series = self._analysis.test[ts_id_cols].value_counts().index + for time_series_id_values in all_time_series: + column_value_combinations = zip( + ts_id_cols, + time_series_id_values) + id_columns_name_value_mapping = [] + filters = [] + for (col, val) in column_value_combinations: + id_columns_name_value_mapping.append(f"{col} = {val}") + filters.append(CohortFilter( + method=CohortFilterMethods.METHOD_INCLUDES, + arg=[val], + column=col)) + time_series = Cohort(", ".join(id_columns_name_value_mapping)) + for filter in filters: + time_series.add_cohort_filter(filter) + cohort_list.append(time_series) + return cohort_list + def _validate_cohort_list(self, cohort_list=None): - if cohort_list is None: + task_type = self.dashboard_input.dataset.task_type + if (task_type != ModelTask.FORECASTING and + cohort_list is None): + self.dashboard_input.cohortData = [] return + if task_type == ModelTask.FORECASTING: + # Ensure user did not pass cohort_list and + # use the generated time series. + if cohort_list is not None: + raise UserConfigValidationException( + "cohort_list is not supported for forecasting analysis.") + # use generated time series + cohort_list = self._generate_time_series_cohorts() + if not isinstance(cohort_list, list): raise UserConfigValidationException( "cohort_list parameter should be a list.") @@ -74,8 +104,7 @@ def _validate_cohort_list(self, cohort_list=None): test_data = pd.DataFrame( data=self.dashboard_input.dataset.features, columns=self.dashboard_input.dataset.feature_names) - if self.dashboard_input.dataset.task_type == \ - ModelTask.CLASSIFICATION: + if task_type == ModelTask.CLASSIFICATION: class_names_list = self.dashboard_input.dataset.class_names true_y_array = self.dashboard_input.dataset.true_y true_class_array = np.array( @@ -95,6 +124,8 @@ def _validate_cohort_list(self, cohort_list=None): categorical_features=categorical_features, is_classification=self._is_classifier) + self.dashboard_input.cohortData = cohort_list + def on_predict(self, data): try: data = pd.DataFrame( @@ -356,6 +387,72 @@ def get_object_detection_metrics(self, post_data): WidgetRequestResponseConstants.data: [] } + def forecast(self, post_data): + # This is a separate function from predict since we apply + # transformations to an entire time series. That enables us + # to only send the transformation information from UI to backend + # rather than having to send the entire time series across. + try: + filters = post_data[0] + composite_filters = post_data[1] + transformation = post_data[2] + filtered_data_df = self._analysis.get_filtered_test_data( + filters=filters, + composite_filters=composite_filters, + include_original_columns_only=True) + + transformation_func = None + # transforming with pandas + if len(transformation) > 0: + op, feature, value = transformation + if op == "add": + def add(x): + return x + float(value) + transformation_func = add + elif op == "subtract": + def subtract(x): + return x - float(value) + transformation_func = subtract + elif op == "multiply": + def multiply(x): + return x * float(value) + transformation_func = multiply + elif op == "divide": + def divide(x): + return x / float(value) + transformation_func = divide + elif op == "change": + def change(x): + return float(value) + transformation_func = change + else: + raise ValueError( + f"An invalid transformation operation {op} " + "was provided.") + + filtered_data_df[feature] = \ + filtered_data_df[feature].map(transformation_func) + + predictions = convert_to_list( + self._analysis.model.forecast(filtered_data_df), + EXP_VIZ_ERR_MSG) + # forecast should return a flat list of predictions + if all([len(p) == 1 for p in predictions]): + predictions = [p[0] for p in predictions] + return { + WidgetRequestResponseConstants.data: predictions + } + except Exception as e: + print(e) + traceback.print_exc() + e_str = _format_exception(e) + return { + WidgetRequestResponseConstants.error: + "Failed to generate forecast for time series, " + "inner error: {}".format(e_str), + WidgetRequestResponseConstants.data: [] + } + def get_question_answering_metrics(self, post_data): """Flask endpoint function to get Model Overview metrics for the Question Answering scenario. diff --git a/raiwidgets/requirements-dev.txt b/raiwidgets/requirements-dev.txt index deb3b4247c..908f590820 100644 --- a/raiwidgets/requirements-dev.txt +++ b/raiwidgets/requirements-dev.txt @@ -10,6 +10,8 @@ wheel fairlearn==0.7.0 ml-wrappers>=0.4.0 +sktime +pmdarima # Jupyter dependency that fails with python 3.6 pywinpty==2.0.2; python_version <= '3.6' and sys_platform == 'win32' diff --git a/raiwidgets/tests/test_fairness_calculations.py b/raiwidgets/tests/test_fairness_calculations.py index 6c468115c2..dbb9782fb8 100644 --- a/raiwidgets/tests/test_fairness_calculations.py +++ b/raiwidgets/tests/test_fairness_calculations.py @@ -17,13 +17,13 @@ @pytest.fixture() def sample_binary_data(): - return np.array([0, 1, 1, 1, 0, 1, 0, 1, 0, 0]),\ + return np.array([0, 1, 1, 1, 0, 1, 0, 1, 0, 0]), \ np.array([0, 1, 1, 1, 1, 1, 1, 0, 0, 0]) @pytest.fixture() def sample_continuous_data(): - return np.array([25, 36, 12, 10, 52, 64, 34, 36, 11, 17]),\ + return np.array([25, 36, 12, 10, 52, 64, 34, 36, 11, 17]), \ np.array([37, 29, 20, 2, 12, 75, 53, 64, 23, 29]) diff --git a/responsibleai/responsibleai/rai_insights/rai_insights.py b/responsibleai/responsibleai/rai_insights/rai_insights.py index dcd98f10b9..6199331b09 100644 --- a/responsibleai/responsibleai/rai_insights/rai_insights.py +++ b/responsibleai/responsibleai/rai_insights/rai_insights.py @@ -482,6 +482,8 @@ def _validate_rai_insights_input_parameters( # We specifically do not advertise for this until we want people to # use it. if kwargs.get(_FORECASTING_RAI_INSIGHTS_ENABLED, False): + print("WARNING: Support for the forecasting task type is not yet " + "stable. Please do not use it except for testing purposes.") valid_tasks.append(ModelTask.FORECASTING.value) if task_type not in valid_tasks: message = (f"Unsupported task type '{task_type}'. " diff --git a/scripts/e2e-widget.js b/scripts/e2e-widget.js index 557a8e6564..e0f72f5996 100644 --- a/scripts/e2e-widget.js +++ b/scripts/e2e-widget.js @@ -18,7 +18,8 @@ const tabularFileNames = [ "responsibleaidashboard-housing-classification-model-debugging", "responsibleaidashboard-diabetes-decision-making", "responsibleaidashboard-housing-decision-making", - "responsibleaidashboard-multiclass-dnn-model-debugging" + "responsibleaidashboard-multiclass-dnn-model-debugging", + "responsibleaidashboard-orange-juice-forecasting" ]; const visionFileNames = [ "responsibleaidashboard-fridge-image-classification-model-debugging",