Loosen xlrd requirement (#572)

* Loosen xlrd requirement * Release notes * Support upgrade by specifying engine * Remove openpyxl from extra requirements * Black * Update RELEASE_NOTES.md Co-authored-by: Daniel Huppmann <[email protected]> Co-authored-by: Daniel Huppmann <[email protected]>
IAMconsortium · Aug 15, 2021 · 4cf3b98 · 4cf3b98
1 parent bd6fb43
commit 4cf3b98
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 8 deletions.
diff --git a/.github/workflows/pytest-dependency.yml b/.github/workflows/pytest-dependency.yml
@@ -25,7 +25,7 @@ jobs:
     - name: Install specific out-dated version of dependencies
       # Update the package requirements when changing minimum dependency versions
       # Please also add a section "Dependency changes" to the release notes
-      run: pip install pandas==1.1.1 numpy==1.19.0 matplotlib==3.2.0 iam-units==2020.4.21
+      run: pip install pandas==1.1.1 numpy==1.19.0 matplotlib==3.2.0 iam-units==2020.4.21 xlrd==2.0
 
     - name: Install other dependencies and package
       run: pip install -e .[tests,deploy,optional-plotting,optional-io-formats,tutorials]

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -1,3 +1,13 @@
+# Next Release
+
+## Dependency changes
+
+The dependencies were updated to require `xlrd>=2.0` (previously `<2.0`) and `openpyxl` was added as a dependency.
+
+## Individual updates
+
+- [#572](https://github.com/IAMconsortium/pyam/pull/572) Unpinned the requirements for xlrd and added openpyxl as a requirement to ensure ongoing support of both `.xlsx` and `.xls` files out of the box
+
 # Release v1.1.0
 
 ## Highlights

diff --git a/pyam/core.py b/pyam/core.py
@@ -194,7 +194,7 @@ def _init(self, data, meta=None, index=DEFAULT_META_INDEX, **kwargs):
 
         # if initializing from xlsx, try to load `meta` table from file
         if meta_sheet and isinstance(data, Path) and data.suffix == ".xlsx":
-            excel_file = pd.ExcelFile(data)
+            excel_file = pd.ExcelFile(data, engine="openpyxl")
             if meta_sheet in excel_file.sheet_names:
                 self.load_meta(excel_file, sheet_name=meta_sheet, ignore_conflict=True)
 

diff --git a/pyam/utils.py b/pyam/utils.py
@@ -138,7 +138,13 @@ def read_pandas(path, sheet_name="data*", *args, **kwargs):
     if isinstance(path, Path) and path.suffix == ".csv":
         return pd.read_csv(path, *args, **kwargs)
     else:
-        xl = pd.ExcelFile(path)
+        if isinstance(path, pd.ExcelFile):
+            xl = path
+        else:
+            xl = pd.ExcelFile(
+                path, engine="xlrd" if path.suffix == ".xls" else "openpyxl"
+            )
+
         sheet_names = pd.Series(xl.sheet_names)
 
         # reading multiple sheets
@@ -153,7 +159,7 @@ def read_pandas(path, sheet_name="data*", *args, **kwargs):
 
         # read single sheet (if only one exists in file) ignoring sheet name
         else:
-            df = pd.read_excel(path, *args, **kwargs)
+            df = pd.read_excel(xl, *args, **kwargs)
 
         # remove unnamed and empty columns, and rows were all values are nan
         def is_empty(name, s):

diff --git a/setup.py b/setup.py
@@ -13,20 +13,21 @@
 
 # NOTE TO DEVS
 # If you change a minimum version below, please explicitly implement the change
-# in our minimum-reqs test in the file ./.github/workflows/pytest-depedency.yml
+# in our minimum-reqs test in the file ./.github/workflows/pytest-dependency.yml
 # Please also add a section "Dependency changes" to the release notes
 REQUIREMENTS = [
     "argparse",
     "iam-units>=2020.4.21",
     "numpy>=1.19.0",
     "requests",
+    "openpyxl",
     "pandas>=1.1.1",
     "pint",
     "PyYAML",
     "matplotlib>=3.2.0",
     "seaborn",
     "six",
-    "xlrd<2.0",
+    "xlrd>=2.0",
 ]
 
 EXTRA_REQUIREMENTS = {
@@ -44,7 +45,6 @@
         "sphinxcontrib-bibtex<2.0",
         "sphinxcontrib-programoutput",
         "numpydoc",
-        "openpyxl",
         "kaleido",
     ]  # docs requires 'tutorials'
     # GitHub Actions requires pandoc explicitly to build the docs

diff --git a/tests/test_io.py b/tests/test_io.py
@@ -106,7 +106,7 @@ def test_init_df_with_na_unit(test_pd_df, tmpdir):
 
     file = tmpdir / "na_unit.xlsx"
     df.to_excel(file)
-    df_excel = pd.read_excel(file)
+    df_excel = pd.read_excel(file, engine="openpyxl")
     assert np.isnan(df_excel.loc[1, "Unit"])
     IamDataFrame(file)  # reading from file as IamDataFrame works