diff --git a/activity_browser/bwutils/errors.py b/activity_browser/bwutils/errors.py index c3eabd3d4..3b4c0d3e1 100644 --- a/activity_browser/bwutils/errors.py +++ b/activity_browser/bwutils/errors.py @@ -96,5 +96,16 @@ class ScenarioExchangeDataNotFoundError(ABError): pass +class ScenarioExchangeDataNonNumericError(ABError): + """Should be raised if non-numeric data is provided for the exchanges in a scenario difference file.""" + pass + + class UnalignableScenarioColumnsWarning(ABWarning): - """Should be raised if there is a mismatch between the scenario columns from multiple scenario difference files""" \ No newline at end of file + """Should be raised if there is a mismatch between the scenario columns from multiple scenario difference files""" + + +class WrongFileTypeImportError(ABError): + """Should be raised when a user tries to import the wrong type of file for the import in question. + For example a database file with the scenario import dialog, or vice versa.""" + pass diff --git a/activity_browser/bwutils/superstructure/__init__.py b/activity_browser/bwutils/superstructure/__init__.py index f704f13d8..69d7c16f8 100644 --- a/activity_browser/bwutils/superstructure/__init__.py +++ b/activity_browser/bwutils/superstructure/__init__.py @@ -6,7 +6,10 @@ from .file_imports import ( ABFeatherImporter, ABCSVImporter, ABFileImporter ) +from .file_dialogs import ( + ABPopup +) from .excel import import_from_excel, get_sheet_names from .manager import SuperstructureManager from .mlca import SuperstructureMLCA, SuperstructureContributions -from .utils import SUPERSTRUCTURE, _time_it_ +from .utils import SUPERSTRUCTURE, _time_it_, edit_superstructure_for_string diff --git a/activity_browser/bwutils/superstructure/dataframe.py b/activity_browser/bwutils/superstructure/dataframe.py index f23bae73e..d593be4ba 100644 --- a/activity_browser/bwutils/superstructure/dataframe.py +++ b/activity_browser/bwutils/superstructure/dataframe.py @@ -187,8 +187,7 @@ def exchange_replace_database(ds: pd.Series, replacements: dict, critical: list, f"
To abort the process press \'Cancel\'" critical_message = ABPopup.abCritical("Activities not found", msg, QPushButton('Save'), QPushButton('Cancel'), default=2) critical_message.save_options() - critical_message.dataframe(pd.DataFrame(critical), - ['from database', 'from activity name', 'to database', 'to activity name']) + critical_message.dataframe(df.loc[critical['index'], :], SUPERSTRUCTURE) critical_message.dataframe_to_file(df_, critical['index']) response = critical_message.exec_() else: @@ -198,8 +197,7 @@ def exchange_replace_database(ds: pd.Series, replacements: dict, critical: list, f" failed relinking.
To abort the process press \'Cancel\'" critical_message = ABPopup.abCritical("Activity not found", msg, QPushButton('Save'), QPushButton('Cancel'), default=2) critical_message.save_options() - critical_message.dataframe(pd.DataFrame(critical), - ['from database', 'from activity name', 'to database', 'to activity name']) + critical_message.dataframe(df.loc[critical['index'], :], SUPERSTRUCTURE) critical_message.dataframe_to_file(df_, critical['index']) response = critical_message.exec_() QApplication.setOverrideCursor(Qt.WaitCursor) diff --git a/activity_browser/bwutils/superstructure/excel.py b/activity_browser/bwutils/superstructure/excel.py index 3980f2316..440c1eb9c 100644 --- a/activity_browser/bwutils/superstructure/excel.py +++ b/activity_browser/bwutils/superstructure/excel.py @@ -69,18 +69,22 @@ def import_from_excel(document_path: Union[str, Path], import_sheet: int = 1) -> 'usecols' is used to exclude specific columns from the excel document. 'comment' is used to exclude specific rows from the excel document. """ - header_idx = get_header_index(document_path, import_sheet) - data = pd.read_excel( - document_path, sheet_name=import_sheet, header=header_idx, - usecols=valid_cols, comment="*", na_values="", keep_default_na=False, - engine="openpyxl" - ) - diff = SUPERSTRUCTURE.difference(data.columns) - if not diff.empty: - raise ValueError("Missing required column(s) for superstructure: {}".format(diff.to_list())) - - # Convert specific columns that may have tuples as strings - columns = ["from categories", "from key", "to categories", "to key"] - data.loc[:, columns] = data[columns].applymap(convert_tuple_str) - - return data \ No newline at end of file + data = pd.DataFrame({}) + try: + header_idx = get_header_index(document_path, import_sheet) + data = pd.read_excel( + document_path, sheet_name=import_sheet, header=header_idx, + usecols=valid_cols, comment="*", na_values="", keep_default_na=False, + engine="openpyxl" + ) + diff = SUPERSTRUCTURE.difference(data.columns) + if not diff.empty: + raise ValueError("Missing required column(s) for superstructure: {}".format(diff.to_list())) + + # Convert specific columns that may have tuples as strings + columns = ["from categories", "from key", "to categories", "to key"] + data.loc[:, columns] = data[columns].applymap(convert_tuple_str) + except: + # skip the error checks here, these now occur in the calling layout.tabs.LCA_setup module + pass + return data diff --git a/activity_browser/bwutils/superstructure/file_dialogs.py b/activity_browser/bwutils/superstructure/file_dialogs.py index b01a8ab49..3b57d899f 100644 --- a/activity_browser/bwutils/superstructure/file_dialogs.py +++ b/activity_browser/bwutils/superstructure/file_dialogs.py @@ -20,6 +20,7 @@ class ProblemDataModel(QtCore.QAbstractTableModel): A simple table model for use in the ABPopup dialogs for error reporting. Intentionally coupled with the ABPopup class and not intended for use externally. + """ updated = QtCore.Signal() def __init__(self): @@ -46,9 +47,13 @@ def sync(self, *args, **kwargs) -> None: assert('dataframe' in kwargs and 'columns in kwargs') self.columns = kwargs['columns'] data = kwargs['dataframe'] - self._dataframe = pd.DataFrame(data,columns=self.columns) + self._dataframe = pd.DataFrame(data, columns=self.columns) self.updated.emit() + def headerData(self, section, orientation, role = QtCore.Qt.DisplayRole): + if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole: + return self.columns[section] + class ProblemDataFrame(QtWidgets.QTableView): """ @@ -116,13 +121,14 @@ def dataframe(self, data: pd.DataFrame, columns: list = None): Arguments --------- - data: the dataframe that generates the error + data: a dataframe with the exchanges/rows that generate the error columns: a list of columns to provide the dataframe with for the popup message """ dataframe = data cols = pd.Index(columns) dataframe = dataframe.loc[:, columns] - dataframe.index = dataframe.index.astype(str) + if not isinstance(dataframe.index, pd.MultiIndex): + dataframe.index = dataframe.index.astype(str) self.data_frame.update(dataframe, cols) self.data_frame.setHidden(False) self.updateGeometry() diff --git a/activity_browser/bwutils/superstructure/manager.py b/activity_browser/bwutils/superstructure/manager.py index 8f15c1b2d..e80f49c1e 100644 --- a/activity_browser/bwutils/superstructure/manager.py +++ b/activity_browser/bwutils/superstructure/manager.py @@ -3,6 +3,7 @@ from typing import List import numpy as np import pandas as pd +from pandas.api.types import is_numeric_dtype, is_number from PySide2.QtWidgets import QApplication, QPushButton from PySide2.QtCore import Qt from typing import Union, Optional @@ -11,7 +12,7 @@ from .activities import fill_df_keys_with_fields, get_activities_from_keys from .dataframe import scenario_columns -from .utils import guess_flow_type, SUPERSTRUCTURE, _time_it_ +from .utils import guess_flow_type, SUPERSTRUCTURE, _time_it_, edit_superstructure_for_string import logging from activity_browser.logger import ABHandler @@ -22,7 +23,7 @@ from .file_dialogs import ABPopup from ..errors import (CriticalScenarioExtensionError, ScenarioExchangeNotFoundError, ImportCanceledError, ScenarioExchangeDataNotFoundError, - UnalignableScenarioColumnsWarning, + UnalignableScenarioColumnsWarning, ScenarioExchangeDataNonNumericError ) @@ -376,7 +377,6 @@ def fill_empty_process_keys_in_exchanges(df: pd.DataFrame) -> pd.DataFrame: _df = df.loc[df.loc[:, EXCHANGE_KEYS].isna().any(axis=1)] if not _df.empty: sdf_keys = SuperstructureManager.exchangesPopup() - sdf_keys.save_options() sdf_keys.dataframe_to_file(df, _df.index) QApplication.restoreOverrideCursor() sdf_keys.exec_() @@ -407,10 +407,8 @@ def verify_scenario_process_keys(df: pd.DataFrame) -> pd.DataFrame: _ = get_activities_from_keys(df, db) df_ = pd.concat([df_, _], axis=0, ignore_index=False) if not df_.empty: - errors_df = pd.DataFrame(df_, index=None, columns=['from key', 'to key', 'flow type']) sdf_keys = SuperstructureManager.exchangesPopup() - sdf_keys.save_options() - sdf_keys.dataframe(errors_df, errors_df.columns) + sdf_keys.dataframe(df_, SUPERSTRUCTURE) sdf_keys.dataframe_to_file(df, df_.index) QApplication.restoreOverrideCursor() sdf_keys.exec_() @@ -426,6 +424,7 @@ def check_scenario_exchange_values(df: pd.DataFrame, cols: pd.Index): Raises ------ A ScenarioExchangeDataNotFoundError if no valid values are found in the scenario 'amounts' + A ScenarioExchangeDataNonNumericError if non-numeric values are found for the scenario 'amounts' A logged warning before replacement of invalid scenario values Parameters @@ -435,8 +434,9 @@ def check_scenario_exchange_values(df: pd.DataFrame, cols: pd.Index): cols: a pandas index that indicates the scenario columns holding the 'amounts' to be used in the scenario calculations """ + _df = df.copy() assert len(cols) > 0 - nas = df.loc[:, cols].isna() + nas = _df.loc[:, cols].isna() if nas.all(axis=0).all(): msg = "

No exchange values could be observed in the last loaded scenario file. " + \ "Exchange values must be recorded in a labelled scenario column with a name distinguishable from the" + \ @@ -450,6 +450,29 @@ def check_scenario_exchange_values(df: pd.DataFrame, cols: pd.Index): raise ScenarioExchangeDataNotFoundError elif nas.any(axis=0).any(): log.warning("Replacing empty values from the last loaded scenario difference file") + if not is_numeric_dtype(np.array(_df.loc[:, cols])): + # converting to numeric only works on lists and with the coercive option + # any errors convert to np.nan and can then only be excluded if previous + # NaNs are masked by conversion to numeric values + _df.loc[:, cols].fillna(0, inplace=True) + bad_entries = pd.DataFrame(index=_df.index) + for col in cols: + bad_entries[col] = pd.to_numeric(df.loc[:, col], errors='coerce') + msg = "

Non-numeric data is present in the scenario exchange columns.

The Activity-Browser can "\ + "only deal with numeric data for the calculations. To resolve this corrections will need to be made "\ + "to these values in the scenario file.

" + critical = ABPopup.abCritical( + "Bad (non-numeric) input data", + msg, + QPushButton('Save'), + QPushButton('Cancel') + ) + critical.dataframe(df[bad_entries.isna().any(axis=1)], SUPERSTRUCTURE) + critical.save_options() + critical.dataframe_to_file(df, bad_entries.isna().any(axis=1)) + QApplication.restoreOverrideCursor() + critical.exec_() + raise ScenarioExchangeDataNonNumericError() @staticmethod @_time_it_ @@ -497,7 +520,7 @@ def check_duplicates(data: Optional[Union[pd.DataFrame, list]], for file, frame in duplicated.items(): frame.insert(0, 'File', file, allow_duplicates=True) warning = ABPopup.abWarning('Duplicate flow exchanges', msg, QPushButton('Ok'), QPushButton('Cancel')) - warning.dataframe(pd.concat([file for file in duplicated.values()]), index) + warning.dataframe(pd.concat([file for file in duplicated.values()]), ['File'] + SUPERSTRUCTURE) QApplication.restoreOverrideCursor() response = warning.exec_() QApplication.setOverrideCursor(Qt.WaitCursor) @@ -532,8 +555,7 @@ def _check_duplicate(data: pd.DataFrame, index: list = ['to key', 'from key', 'f "scenarios for the same flow. The AB can deal with this by discarding all but the last row for this " \ "exchange.

Press 'Ok' to proceed, press 'Cancel' to abort.

" warning = ABPopup.abWarning('Duplicate flow exchanges', msg, QPushButton('Ok'), QPushButton('Cancel')) - warning.dataframe(df.loc[duplicates], index) - + warning.dataframe(df.loc[duplicates], SUPERSTRUCTURE) QApplication.restoreOverrideCursor() response = warning.exec_() QApplication.setOverrideCursor(Qt.WaitCursor) @@ -542,11 +564,3 @@ def _check_duplicate(data: pd.DataFrame, index: list = ['to key', 'from key', 'f data.drop_duplicates(index, keep='last', inplace=True) return data - @staticmethod - def edit_superstructure_for_string(): - text_list = "" - for field in SUPERSTRUCTURE: - text_list+= f"{field}
" - return text_list - - diff --git a/activity_browser/bwutils/superstructure/utils.py b/activity_browser/bwutils/superstructure/utils.py index 77cec5538..a733f59f5 100644 --- a/activity_browser/bwutils/superstructure/utils.py +++ b/activity_browser/bwutils/superstructure/utils.py @@ -27,6 +27,28 @@ "flow type", ]) +def edit_superstructure_for_string(superstructure=SUPERSTRUCTURE, sep="
", fhighlight=""): + """ + Produces a string format for the essential columns for the scenario difference files with html + style formatting. Allows for different defined structures. + + Parameters + ---------- + superstructure: the list of superstructure column headers (by default set to the SUPERSTRUCTURE index, + this needs to have a defined __str__ operator + sep: a short string that defines the separator for the column headers, by default this is the html line + break
+ fhighlight: this is provided as a means to highlight the fields, by default this is empty (SHOULD NOT BE + SET TO None), but could be set to "[]", where the first and last elements enclose the field + + Returns + ------- + A formatted strign with the required file fields + """ + text_list = "" + for field in superstructure: + text_list += f"{fhighlight[0]}{field}{fhighlight[-1]}{sep}" if fhighlight else f"{field}{sep}" + return text_list def guess_flow_type(row: pd.Series) -> str: """Given a series of input- and output keys, make a guess on the flow type. @@ -47,4 +69,4 @@ def wrapper(*args): result = func(*args) log.info(f"{func} -- " + str(time.time() - now)) return result - return wrapper \ No newline at end of file + return wrapper diff --git a/activity_browser/layouts/tabs/LCA_setup.py b/activity_browser/layouts/tabs/LCA_setup.py index 0fd93eb7c..5af41c4eb 100644 --- a/activity_browser/layouts/tabs/LCA_setup.py +++ b/activity_browser/layouts/tabs/LCA_setup.py @@ -17,7 +17,7 @@ from ...settings import ab_settings from ...bwutils.errors import (CriticalScenarioExtensionError, ScenarioExchangeNotFoundError, ScenarioDatabaseNotFoundError, ImportCanceledError, ScenarioExchangeDataNotFoundError, - UnalignableScenarioColumnsWarning,) + UnalignableScenarioColumnsWarning, ScenarioExchangeDataNonNumericError,) from ...signals import signals from ...ui.icons import qicons from ...ui.style import horizontal_line, header, style_group_box @@ -26,6 +26,7 @@ ) from ...ui.widgets import ExcelReadDialog, ScenarioDatabaseDialog from .base import BaseRightTab +from activity_browser.bwutils.superstructure import ABPopup, edit_superstructure_for_string import logging from activity_browser.logger import ABHandler @@ -542,6 +543,19 @@ def load_action(self) -> None: if query == QtWidgets.QMessageBox.No: include_default = False signals.parameter_scenario_sync.emit(self.index, df, include_default) + else: + # this is a wrong file type + msg = "The Activity-Browser is attempting to import a scenario file.

During the attempted import"\ + " another file type was detected. Please check the file type of the attempted import, if it is"\ + " a scenario file make sure it contains a valid format.

"\ + "

A flow exchange scenario file requires the following headers:
" +\ + edit_superstructure_for_string(sep=", ", fhighlight='"') + "

"\ + "

A parameter scenario file requires the following:
" + edit_superstructure_for_string( + ["name", "group"], sep=", ", fhighlight='"') + "

" + critical = ABPopup.abCritical("Wrong file type", msg, QtWidgets.QPushButton("Cancel")) + QtWidgets.QApplication.restoreOverrideCursor() + critical.exec_() + return except CriticalScenarioExtensionError as e: # Triggered when combining different scenario files by extension leads to no scenario columns QtWidgets.QApplication.restoreOverrideCursor()