From 03d3f805870b33fcf92088ca2c1623eaae6c6cae Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Thu, 30 Jun 2022 13:10:23 +0200 Subject: [PATCH 1/7] basic migration --- pyam/core.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pyam/core.py b/pyam/core.py index 0a18a8f8f..f84f2126b 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -204,8 +204,7 @@ def _init(self, data, meta=None, index=DEFAULT_META_INDEX, **kwargs): # if given explicitly, merge meta dataframe after downselecting if meta is not None: - meta = meta.loc[self.meta.index.intersection(meta.index)] - self.meta = merge_meta(meta, self.meta, ignore_conflict=True) + self.set_meta(meta) # if initializing from xlsx, try to load `meta` table from file if meta_sheet and isinstance(data, Path) and data.suffix == ".xlsx": @@ -808,7 +807,7 @@ def set_meta(self, meta, name=None, index=None): Parameters ---------- - meta : pandas.Series, list, int, float or str + meta : pandas.DataFrame, pandas.Series, list, int, float or str column to be added to 'meta' (by `['model', 'scenario']` index if possible) name : str, optional @@ -817,6 +816,11 @@ def set_meta(self, meta, name=None, index=None): index : IamDataFrame, pandas.DataFrame or pandas.MultiIndex, optional index to be used for setting meta column (`['model', 'scenario']`) """ + if isinstance(meta, pd.DataFrame): + meta = meta.loc[self.meta.index.intersection(meta.index)] + self.meta = merge_meta(meta, self.meta, ignore_conflict=True) + return + # check that name is valid and doesn't conflict with data columns if (name or (hasattr(meta, "name") and meta.name)) in [None, False]: raise ValueError("Must pass a name or use a named pd.Series") From 9e2cf1264d049bc5a77314c7047790ef80a1752d Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Thu, 30 Jun 2022 13:28:32 +0200 Subject: [PATCH 2/7] migrate all uses to set_meta --- pyam/core.py | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/pyam/core.py b/pyam/core.py index f84f2126b..8b3c78a38 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -817,6 +817,8 @@ def set_meta(self, meta, name=None, index=None): index to be used for setting meta column (`['model', 'scenario']`) """ if isinstance(meta, pd.DataFrame): + if meta.index.names != self.meta.index.names: + meta = meta.set_index(self.meta.index.names) meta = meta.loc[self.meta.index.intersection(meta.index)] self.meta = merge_meta(meta, self.meta, ignore_conflict=True) return @@ -2422,32 +2424,7 @@ def load_meta( f"missing required index columns {missing_cols}!" ) - # set index, filter to relevant scenarios from imported file - n = len(df) - df.set_index(self.index.names, inplace=True) - df = df.loc[self.meta.index.intersection(df.index)] - - # skip import of meta indicators if np - if not n: - logger.info(f"No scenarios found in sheet {sheet_name}") - return - - msg = "Reading meta indicators" - # indicate if not all scenarios are included in the meta file - if len(df) < len(self.meta): - i = len(self.meta) - msg += f" for {len(df)} out of {i} scenario{s(i)}" - - # indicate if more scenarios exist in meta file than in self - invalid = n - len(df) - if invalid: - msg += f", ignoring {invalid} scenario{s(invalid)} from file" - logger.warning(msg) - else: - logger.info(msg) - - # merge imported meta indicators - self.meta = merge_meta(df, self.meta, ignore_conflict=ignore_conflict) + self.set_meta(df) def map_regions( self, From ea199370b9b2adf06b7289c304142b01baeff6b7 Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Thu, 30 Jun 2022 13:31:28 +0200 Subject: [PATCH 3/7] add readme --- RELEASE_NOTES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 3dddf36e1..9406bb7eb 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,9 @@ # Next Release +## Individual updates + +- [#679](https://github.com/IAMconsortium/pyam/pull/679) `set_meta()` now supports pandas.DataFrame as an argument + # Release v1.5.0 ## Highlights From ce331256e075e2f32ff2d62a5b203b1baef0a759 Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Fri, 1 Jul 2022 11:32:47 +0200 Subject: [PATCH 4/7] revert removal of warnings --- pyam/core.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/pyam/core.py b/pyam/core.py index 15eb3c304..0f3f17383 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -2443,7 +2443,32 @@ def load_meta( f"missing required index columns {missing_cols}!" ) - self.set_meta(df) + # set index, filter to relevant scenarios from imported file + n = len(df) + df.set_index(self.index.names, inplace=True) + df = df.loc[self.meta.index.intersection(df.index)] + + # skip import of meta indicators if np + if not n: + logger.info(f"No scenarios found in sheet {sheet_name}") + return + + msg = "Reading meta indicators" + # indicate if not all scenarios are included in the meta file + if len(df) < len(self.meta): + i = len(self.meta) + msg += f" for {len(df)} out of {i} scenario{s(i)}" + + # indicate if more scenarios exist in meta file than in self + invalid = n - len(df) + if invalid: + msg += f", ignoring {invalid} scenario{s(invalid)} from file" + logger.warning(msg) + else: + logger.info(msg) + + # merge imported meta indicators + self.meta = merge_meta(df, self.meta, ignore_conflict=ignore_conflict) def map_regions( self, From bbf605e062116ac552c49b342a24178b0861ba4f Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Fri, 1 Jul 2022 17:39:49 +0200 Subject: [PATCH 5/7] catch capital Model and Scenario --- pyam/core.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyam/core.py b/pyam/core.py index 0f3f17383..4bf0216f1 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -818,7 +818,12 @@ def set_meta(self, meta, name=None, index=None): """ if isinstance(meta, pd.DataFrame): if meta.index.names != self.meta.index.names: - meta = meta.set_index(self.meta.index.names) + # catch Model, Scenario instead of model, scenario + meta = ( + meta + .rename(columns={i.capitalize(): i for i in META_IDX}) + .set_index(self.meta.index.names) + ) meta = meta.loc[self.meta.index.intersection(meta.index)] self.meta = merge_meta(meta, self.meta, ignore_conflict=True) return From ad03e0c6d66d8c4f8d1efe3c8dfbf1f31e0a42c0 Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Fri, 8 Jul 2022 09:23:10 +0200 Subject: [PATCH 6/7] stickler --- pyam/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyam/core.py b/pyam/core.py index 4bf0216f1..f952b5e5d 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -821,7 +821,7 @@ def set_meta(self, meta, name=None, index=None): # catch Model, Scenario instead of model, scenario meta = ( meta - .rename(columns={i.capitalize(): i for i in META_IDX}) + .rename(columns={i.capitalize(): i for i in META_IDX}) .set_index(self.meta.index.names) ) meta = meta.loc[self.meta.index.intersection(meta.index)] From 776494b303dfd240c3a0d0e1c5c23d9dd3e090d9 Mon Sep 17 00:00:00 2001 From: Matthew Gidden Date: Fri, 8 Jul 2022 09:25:36 +0200 Subject: [PATCH 7/7] surrender to black, for now --- pyam/core.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pyam/core.py b/pyam/core.py index f952b5e5d..5a41c6377 100755 --- a/pyam/core.py +++ b/pyam/core.py @@ -819,11 +819,9 @@ def set_meta(self, meta, name=None, index=None): if isinstance(meta, pd.DataFrame): if meta.index.names != self.meta.index.names: # catch Model, Scenario instead of model, scenario - meta = ( - meta - .rename(columns={i.capitalize(): i for i in META_IDX}) - .set_index(self.meta.index.names) - ) + meta = meta.rename( + columns={i.capitalize(): i for i in META_IDX} + ).set_index(self.meta.index.names) meta = meta.loc[self.meta.index.intersection(meta.index)] self.meta = merge_meta(meta, self.meta, ignore_conflict=True) return