diff --git a/activity_browser/bwutils/errors.py b/activity_browser/bwutils/errors.py
index c3eabd3d4..3b4c0d3e1 100644
--- a/activity_browser/bwutils/errors.py
+++ b/activity_browser/bwutils/errors.py
@@ -96,5 +96,16 @@ class ScenarioExchangeDataNotFoundError(ABError):
pass
+class ScenarioExchangeDataNonNumericError(ABError):
+ """Should be raised if non-numeric data is provided for the exchanges in a scenario difference file."""
+ pass
+
+
class UnalignableScenarioColumnsWarning(ABWarning):
- """Should be raised if there is a mismatch between the scenario columns from multiple scenario difference files"""
\ No newline at end of file
+ """Should be raised if there is a mismatch between the scenario columns from multiple scenario difference files"""
+
+
+class WrongFileTypeImportError(ABError):
+ """Should be raised when a user tries to import the wrong type of file for the import in question.
+ For example a database file with the scenario import dialog, or vice versa."""
+ pass
diff --git a/activity_browser/bwutils/superstructure/__init__.py b/activity_browser/bwutils/superstructure/__init__.py
index f704f13d8..69d7c16f8 100644
--- a/activity_browser/bwutils/superstructure/__init__.py
+++ b/activity_browser/bwutils/superstructure/__init__.py
@@ -6,7 +6,10 @@
from .file_imports import (
ABFeatherImporter, ABCSVImporter, ABFileImporter
)
+from .file_dialogs import (
+ ABPopup
+)
from .excel import import_from_excel, get_sheet_names
from .manager import SuperstructureManager
from .mlca import SuperstructureMLCA, SuperstructureContributions
-from .utils import SUPERSTRUCTURE, _time_it_
+from .utils import SUPERSTRUCTURE, _time_it_, edit_superstructure_for_string
diff --git a/activity_browser/bwutils/superstructure/dataframe.py b/activity_browser/bwutils/superstructure/dataframe.py
index f23bae73e..d593be4ba 100644
--- a/activity_browser/bwutils/superstructure/dataframe.py
+++ b/activity_browser/bwutils/superstructure/dataframe.py
@@ -187,8 +187,7 @@ def exchange_replace_database(ds: pd.Series, replacements: dict, critical: list,
f"
To abort the process press \'Cancel\'"
critical_message = ABPopup.abCritical("Activities not found", msg, QPushButton('Save'), QPushButton('Cancel'), default=2)
critical_message.save_options()
- critical_message.dataframe(pd.DataFrame(critical),
- ['from database', 'from activity name', 'to database', 'to activity name'])
+ critical_message.dataframe(df.loc[critical['index'], :], SUPERSTRUCTURE)
critical_message.dataframe_to_file(df_, critical['index'])
response = critical_message.exec_()
else:
@@ -198,8 +197,7 @@ def exchange_replace_database(ds: pd.Series, replacements: dict, critical: list,
f" failed relinking.
To abort the process press \'Cancel\'"
critical_message = ABPopup.abCritical("Activity not found", msg, QPushButton('Save'), QPushButton('Cancel'), default=2)
critical_message.save_options()
- critical_message.dataframe(pd.DataFrame(critical),
- ['from database', 'from activity name', 'to database', 'to activity name'])
+ critical_message.dataframe(df.loc[critical['index'], :], SUPERSTRUCTURE)
critical_message.dataframe_to_file(df_, critical['index'])
response = critical_message.exec_()
QApplication.setOverrideCursor(Qt.WaitCursor)
diff --git a/activity_browser/bwutils/superstructure/excel.py b/activity_browser/bwutils/superstructure/excel.py
index 3980f2316..440c1eb9c 100644
--- a/activity_browser/bwutils/superstructure/excel.py
+++ b/activity_browser/bwutils/superstructure/excel.py
@@ -69,18 +69,22 @@ def import_from_excel(document_path: Union[str, Path], import_sheet: int = 1) ->
'usecols' is used to exclude specific columns from the excel document.
'comment' is used to exclude specific rows from the excel document.
"""
- header_idx = get_header_index(document_path, import_sheet)
- data = pd.read_excel(
- document_path, sheet_name=import_sheet, header=header_idx,
- usecols=valid_cols, comment="*", na_values="", keep_default_na=False,
- engine="openpyxl"
- )
- diff = SUPERSTRUCTURE.difference(data.columns)
- if not diff.empty:
- raise ValueError("Missing required column(s) for superstructure: {}".format(diff.to_list()))
-
- # Convert specific columns that may have tuples as strings
- columns = ["from categories", "from key", "to categories", "to key"]
- data.loc[:, columns] = data[columns].applymap(convert_tuple_str)
-
- return data
\ No newline at end of file
+ data = pd.DataFrame({})
+ try:
+ header_idx = get_header_index(document_path, import_sheet)
+ data = pd.read_excel(
+ document_path, sheet_name=import_sheet, header=header_idx,
+ usecols=valid_cols, comment="*", na_values="", keep_default_na=False,
+ engine="openpyxl"
+ )
+ diff = SUPERSTRUCTURE.difference(data.columns)
+ if not diff.empty:
+ raise ValueError("Missing required column(s) for superstructure: {}".format(diff.to_list()))
+
+ # Convert specific columns that may have tuples as strings
+ columns = ["from categories", "from key", "to categories", "to key"]
+ data.loc[:, columns] = data[columns].applymap(convert_tuple_str)
+ except:
+ # skip the error checks here, these now occur in the calling layout.tabs.LCA_setup module
+ pass
+ return data
diff --git a/activity_browser/bwutils/superstructure/file_dialogs.py b/activity_browser/bwutils/superstructure/file_dialogs.py
index b01a8ab49..3b57d899f 100644
--- a/activity_browser/bwutils/superstructure/file_dialogs.py
+++ b/activity_browser/bwutils/superstructure/file_dialogs.py
@@ -20,6 +20,7 @@ class ProblemDataModel(QtCore.QAbstractTableModel):
A simple table model for use in the ABPopup dialogs for error reporting.
Intentionally coupled with the ABPopup class and not intended for use externally.
+
"""
updated = QtCore.Signal()
def __init__(self):
@@ -46,9 +47,13 @@ def sync(self, *args, **kwargs) -> None:
assert('dataframe' in kwargs and 'columns in kwargs')
self.columns = kwargs['columns']
data = kwargs['dataframe']
- self._dataframe = pd.DataFrame(data,columns=self.columns)
+ self._dataframe = pd.DataFrame(data, columns=self.columns)
self.updated.emit()
+ def headerData(self, section, orientation, role = QtCore.Qt.DisplayRole):
+ if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
+ return self.columns[section]
+
class ProblemDataFrame(QtWidgets.QTableView):
"""
@@ -116,13 +121,14 @@ def dataframe(self, data: pd.DataFrame, columns: list = None):
Arguments
---------
- data: the dataframe that generates the error
+ data: a dataframe with the exchanges/rows that generate the error
columns: a list of columns to provide the dataframe with for the popup message
"""
dataframe = data
cols = pd.Index(columns)
dataframe = dataframe.loc[:, columns]
- dataframe.index = dataframe.index.astype(str)
+ if not isinstance(dataframe.index, pd.MultiIndex):
+ dataframe.index = dataframe.index.astype(str)
self.data_frame.update(dataframe, cols)
self.data_frame.setHidden(False)
self.updateGeometry()
diff --git a/activity_browser/bwutils/superstructure/manager.py b/activity_browser/bwutils/superstructure/manager.py
index 8f15c1b2d..e80f49c1e 100644
--- a/activity_browser/bwutils/superstructure/manager.py
+++ b/activity_browser/bwutils/superstructure/manager.py
@@ -3,6 +3,7 @@
from typing import List
import numpy as np
import pandas as pd
+from pandas.api.types import is_numeric_dtype, is_number
from PySide2.QtWidgets import QApplication, QPushButton
from PySide2.QtCore import Qt
from typing import Union, Optional
@@ -11,7 +12,7 @@
from .activities import fill_df_keys_with_fields, get_activities_from_keys
from .dataframe import scenario_columns
-from .utils import guess_flow_type, SUPERSTRUCTURE, _time_it_
+from .utils import guess_flow_type, SUPERSTRUCTURE, _time_it_, edit_superstructure_for_string
import logging
from activity_browser.logger import ABHandler
@@ -22,7 +23,7 @@
from .file_dialogs import ABPopup
from ..errors import (CriticalScenarioExtensionError, ScenarioExchangeNotFoundError,
ImportCanceledError, ScenarioExchangeDataNotFoundError,
- UnalignableScenarioColumnsWarning,
+ UnalignableScenarioColumnsWarning, ScenarioExchangeDataNonNumericError
)
@@ -376,7 +377,6 @@ def fill_empty_process_keys_in_exchanges(df: pd.DataFrame) -> pd.DataFrame:
_df = df.loc[df.loc[:, EXCHANGE_KEYS].isna().any(axis=1)]
if not _df.empty:
sdf_keys = SuperstructureManager.exchangesPopup()
- sdf_keys.save_options()
sdf_keys.dataframe_to_file(df, _df.index)
QApplication.restoreOverrideCursor()
sdf_keys.exec_()
@@ -407,10 +407,8 @@ def verify_scenario_process_keys(df: pd.DataFrame) -> pd.DataFrame:
_ = get_activities_from_keys(df, db)
df_ = pd.concat([df_, _], axis=0, ignore_index=False)
if not df_.empty:
- errors_df = pd.DataFrame(df_, index=None, columns=['from key', 'to key', 'flow type'])
sdf_keys = SuperstructureManager.exchangesPopup()
- sdf_keys.save_options()
- sdf_keys.dataframe(errors_df, errors_df.columns)
+ sdf_keys.dataframe(df_, SUPERSTRUCTURE)
sdf_keys.dataframe_to_file(df, df_.index)
QApplication.restoreOverrideCursor()
sdf_keys.exec_()
@@ -426,6 +424,7 @@ def check_scenario_exchange_values(df: pd.DataFrame, cols: pd.Index):
Raises
------
A ScenarioExchangeDataNotFoundError if no valid values are found in the scenario 'amounts'
+ A ScenarioExchangeDataNonNumericError if non-numeric values are found for the scenario 'amounts'
A logged warning before replacement of invalid scenario values
Parameters
@@ -435,8 +434,9 @@ def check_scenario_exchange_values(df: pd.DataFrame, cols: pd.Index):
cols: a pandas index that indicates the scenario columns holding the 'amounts' to be used in the scenario
calculations
"""
+ _df = df.copy()
assert len(cols) > 0
- nas = df.loc[:, cols].isna()
+ nas = _df.loc[:, cols].isna()
if nas.all(axis=0).all():
msg = "
No exchange values could be observed in the last loaded scenario file. " + \ "Exchange values must be recorded in a labelled scenario column with a name distinguishable from the" + \ @@ -450,6 +450,29 @@ def check_scenario_exchange_values(df: pd.DataFrame, cols: pd.Index): raise ScenarioExchangeDataNotFoundError elif nas.any(axis=0).any(): log.warning("Replacing empty values from the last loaded scenario difference file") + if not is_numeric_dtype(np.array(_df.loc[:, cols])): + # converting to numeric only works on lists and with the coercive option + # any errors convert to np.nan and can then only be excluded if previous + # NaNs are masked by conversion to numeric values + _df.loc[:, cols].fillna(0, inplace=True) + bad_entries = pd.DataFrame(index=_df.index) + for col in cols: + bad_entries[col] = pd.to_numeric(df.loc[:, col], errors='coerce') + msg = "
Non-numeric data is present in the scenario exchange columns.
The Activity-Browser can "\ + "only deal with numeric data for the calculations. To resolve this corrections will need to be made "\ + "to these values in the scenario file.
" + critical = ABPopup.abCritical( + "Bad (non-numeric) input data", + msg, + QPushButton('Save'), + QPushButton('Cancel') + ) + critical.dataframe(df[bad_entries.isna().any(axis=1)], SUPERSTRUCTURE) + critical.save_options() + critical.dataframe_to_file(df, bad_entries.isna().any(axis=1)) + QApplication.restoreOverrideCursor() + critical.exec_() + raise ScenarioExchangeDataNonNumericError() @staticmethod @_time_it_ @@ -497,7 +520,7 @@ def check_duplicates(data: Optional[Union[pd.DataFrame, list]], for file, frame in duplicated.items(): frame.insert(0, 'File', file, allow_duplicates=True) warning = ABPopup.abWarning('Duplicate flow exchanges', msg, QPushButton('Ok'), QPushButton('Cancel')) - warning.dataframe(pd.concat([file for file in duplicated.values()]), index) + warning.dataframe(pd.concat([file for file in duplicated.values()]), ['File'] + SUPERSTRUCTURE) QApplication.restoreOverrideCursor() response = warning.exec_() QApplication.setOverrideCursor(Qt.WaitCursor) @@ -532,8 +555,7 @@ def _check_duplicate(data: pd.DataFrame, index: list = ['to key', 'from key', 'f "scenarios for the same flow. The AB can deal with this by discarding all but the last row for this " \ "exchange.Press 'Ok' to proceed, press 'Cancel' to abort.
" warning = ABPopup.abWarning('Duplicate flow exchanges', msg, QPushButton('Ok'), QPushButton('Cancel')) - warning.dataframe(df.loc[duplicates], index) - + warning.dataframe(df.loc[duplicates], SUPERSTRUCTURE) QApplication.restoreOverrideCursor() response = warning.exec_() QApplication.setOverrideCursor(Qt.WaitCursor) @@ -542,11 +564,3 @@ def _check_duplicate(data: pd.DataFrame, index: list = ['to key', 'from key', 'f data.drop_duplicates(index, keep='last', inplace=True) return data - @staticmethod - def edit_superstructure_for_string(): - text_list = "" - for field in SUPERSTRUCTURE: - text_list+= f"{field}During the attempted import"\ + " another file type was detected. Please check the file type of the attempted import, if it is"\ + " a scenario file make sure it contains a valid format.
"\ + "A flow exchange scenario file requires the following headers:
" +\
+ edit_superstructure_for_string(sep=", ", fhighlight='"') + "
A parameter scenario file requires the following:
" + edit_superstructure_for_string(
+ ["name", "group"], sep=", ", fhighlight='"') + "