From d77f44a7c2f2cf93a996e2245a4c0c78a7fa9cf2 Mon Sep 17 00:00:00 2001 From: Leopold Talirz Date: Wed, 17 Oct 2018 11:36:19 +0200 Subject: [PATCH 1/4] Fix conda tests (#37) * fix length of config file name which could enable install on conda forge * update yapf and prospector * disable useless-object-inheritance warning cannot make pylint happy both in python2 and 3 * fix python3 warnings + try to upgrade versions * try fixing version updater split setup.py into setup.py and setup.json (logic was partly written for setup.json, was trying to load setup.py as json file) * fix package_data bug * try fixing install issues * include setup.json in manifest * switch to local prospector installation guykisel mirror was using v0.8 which was breaking pylint on py2.7 * fix prospector version later versions have incorrect handling of 'quiet' flag for pylint<2 See https://github.com/PyCQA/prospector/issues/266 * fix prospector verison * updated: remove obsolete setup_version_pat --- .pre-commit-config.yaml | 12 ++++++------ .pylintrc | 2 +- MANIFEST.in | 1 + ops/update_version.py | 38 +++++++++++++++++++++----------------- reentry/config.py | 12 ++++++++++-- reentry/jsonbackend.py | 15 +++++++++------ setup.json | 31 +++++++++++++++++++++++++++++++ setup.py | 32 +++++++++----------------------- 8 files changed, 88 insertions(+), 55 deletions(-) create mode 100644 setup.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2476b07..3176391 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,18 +1,18 @@ repos: - repo: git://github.com/pre-commit/mirrors-yapf - sha: v0.21.0 + sha: v0.24.0 hooks: - id: yapf language: system -- repo: git://github.com/guykisel/prospector-mirror - sha: b27f281eb9398fc8504415d7fbdabf119ea8c5e1 +- repo: local hooks: - id: prospector language: system - -- repo: local - hooks: + types: [file, python] + name: prospector + description: "This hook runs Prospector: https://github.com/landscapeio/prospector" + entry: prospector - id: travis-linter name: travis entry: travis lint diff --git a/.pylintrc b/.pylintrc index 2a604b9..cfcb0ca 100644 --- a/.pylintrc +++ b/.pylintrc @@ -50,7 +50,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call +disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,useless-object-inheritance # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/MANIFEST.in b/MANIFEST.in index 8925d0c..be246df 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include README.rst include LICENSE +include setup.json recursive-include reentry tests/*json diff --git a/ops/update_version.py b/ops/update_version.py index 97a75d8..2787b3f 100644 --- a/ops/update_version.py +++ b/ops/update_version.py @@ -6,6 +6,7 @@ import fileinput import contextlib import subprocess +import collections from packaging import version from py import path as py_path # pylint: disable=no-name-in-module,no-member @@ -42,13 +43,12 @@ class VersionUpdater(object): version_pat = re.compile(r'\d+.\d+.\d+(-(alpha|beta|rc)(.\d+){0,3}){0,1}') init_version_pat = re.compile(r'(__version__ = )([\'"])(.*?)([\'"])', re.DOTALL | re.MULTILINE) - setup_version_pat = re.compile(r'(VERSION = )([\'"])(.*?)([\'"])', re.DOTALL | re.MULTILINE) replace_tmpl = r'\1\g<2>{}\4' def __init__(self): """Initialize with documents that should be kept up to date and actual version.""" self.top_level_init = py_path.local(subpath('reentry', '__init__.py')) - self.setup_py = py_path.local(subpath('setup.py')) + self.setup_json = py_path.local(subpath('setup.json')) self.version = self.get_version() def write_to_init(self): @@ -56,10 +56,12 @@ def write_to_init(self): self.top_level_init.write(re.sub(self.init_version_pat, self.new_version_str, init_content, re.DOTALL | re.MULTILINE)) def write_to_setup(self): - """Write the updated version number to the setup file.""" - setup_content = self.setup_py.read() - new_content = re.sub(self.setup_version_pat, self.new_version_str, setup_content, re.DOTALL | re.MULTILINE) - self.setup_py.write(new_content) + """Write the updated version number to setup.json.""" + with open(str(self.setup_json), 'r+') as setup_fo: + # preserve order + setup = json.load(setup_fo, object_pairs_hook=collections.OrderedDict) + setup['version'] = self.new_version_str + json.dump(setup, setup_fo, indent=2) @property def new_version_str(self): @@ -67,11 +69,16 @@ def new_version_str(self): @property def setup_version(self): - """Grab the parsed version from the setup file.""" - match = re.search(self.setup_version_pat, self.setup_py.read()) - if not match: - raise AttributeError('No global variable VERSION found in setup.py') - return version.parse(match.groups()[2]) + """Grab the parsed version from setup.json.""" + with open(str(self.setup_json), 'r') as setup_fo: + setup = json.load(setup_fo) + + try: + version_string = setup['version'] + except KeyError: + raise AttributeError('No version found in setup.json') + + return version.parse(version_string) @property def init_version(self): @@ -83,17 +90,14 @@ def init_version(self): @property def tag_version(self): - """Get the current version number from ``git describe``, fall back to setup.py.""" + """Get the current version number from ``git describe``, fall back to setup.json.""" try: describe_byte_string = subprocess.check_output(['git', 'describe', '--tags', '--match', 'v*.*.*']) match = re.search(self.version_pat, describe_byte_string.decode(encoding='UTF-8')) version_string = match.string[match.pos:match.end()] + return version.parse(version_string) except subprocess.CalledProcessError: - with open(self.setup_py, 'r') as setup_fo: - setup = json.load(setup_fo) - version_string = setup['version'] - - return version.parse(version_string) + return self.setup_version def get_version(self): return max(self.setup_version, self.init_version, self.tag_version) diff --git a/reentry/config.py b/reentry/config.py index 1ad78e2..d272b22 100644 --- a/reentry/config.py +++ b/reentry/config.py @@ -1,6 +1,7 @@ """Find and read user settings.""" import os import sys +import hashlib import six from six.moves import configparser @@ -19,6 +20,7 @@ def find_config(): home = py_path.local(os.path.expanduser('~')) rc_file = home.join('.reentryrc') config_file = home.join('.config', 'reentry', 'config') + # pylint: disable=no-else-return if home.exists(): return rc_file elif config_file.exists(): @@ -28,6 +30,7 @@ def find_config(): def make_config_parser(*args, **kwargs): """Get the correct ConfigParser class depending on python version.""" + # pylint: disable=no-else-return if six.PY2: return configparser.SafeConfigParser(*args, **kwargs) elif six.PY3: @@ -53,7 +56,10 @@ def get_config(config_file_name=find_config().strpath): def make_data_file_name(): - """Find the path to the reentry executable and mangle it into a file name.""" + """Find the path to the reentry executable and mangle it into a file name. + + Note: In order to avoid long filenames (e.g. on conda forge), the relevant info is hashed. + """ sep = os.path.sep python_bin_dir = py_path.local(sys.executable).dirname py_version = 'UNKNOWN' @@ -62,7 +68,9 @@ def make_data_file_name(): elif six.PY3: py_version = 'PY3' file_name = python_bin_dir.lstrip(sep).replace(sep, '.').replace('.', '_') + '_' + py_version - return file_name + + file_name_hash = hashlib.sha256(file_name.encode('utf-8')) + return file_name_hash.hexdigest() def get_datafile(): diff --git a/reentry/jsonbackend.py b/reentry/jsonbackend.py index e00ca92..c4f1239 100644 --- a/reentry/jsonbackend.py +++ b/reentry/jsonbackend.py @@ -116,16 +116,18 @@ def get_ep(self, group, name, dist=None): spc = self.get_ep(group, name, dist=dist_name) if spc: specs.append(spc) + # pylint: disable=no-else-return if len(specs) > 1: return specs elif len(specs) == 1: return specs[0] - else: - distribution_map = self._epmap.get(dist, {}) - group_map = distribution_map.get(group, {}) - spec = group_map.get(name) - if spec: - return EntryPoint.parse(spec) + + distribution_map = self._epmap.get(dist, {}) + group_map = distribution_map.get(group, {}) + spec = group_map.get(name) + if spec: + return EntryPoint.parse(spec) + return None def get_dist_names(self): @@ -284,6 +286,7 @@ def _flat_entry_points(self): def _listify(sequence_or_name): """Wrap a single name in a list, leave sequences and None unchanged""" from collections import Sequence + # pylint: disable=no-else-return if sequence_or_name is None: return None elif not isinstance(sequence_or_name, Sequence) or isinstance(sequence_or_name, six.string_types): diff --git a/setup.json b/setup.json new file mode 100644 index 0000000..b546bf2 --- /dev/null +++ b/setup.json @@ -0,0 +1,31 @@ +{ + "name": "reentry", + "version": "1.2.1", + "author": "Rico Haeuselmann", + "license": "MIT License", + "description": "A plugin manager based on setuptools entry points mechanism", + "entry_points": { + "distutils.setup_keywords": [ + "reentry_register = reentry.hooks:register_dist", + "reentry_scan = reentry.hooks:scan_for_installed" + ], + "console_scripts": ["reentry = reentry.cli:reentry"], + "test_entry_points": ["test = reentry.cli:reentry"] + }, + "classifiers":[ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", + "Development Status :: 5 - Production/Stable", + "Environment :: Plugins", + "Intended Audience :: Developers", + "Topic :: Software Development" + ], + "install_requires": ["setuptools >= 18.5", "click", "six", "py"], + "extras_require": { + "dev": ["pre-commit", "prospector==0.12.11", + "pylint", "flake8", "pytest", + "yapf", "coverage", "pytest-cov", + "tox", "packaging"] + } +} diff --git a/setup.py b/setup.py index cddf024..3fffd84 100644 --- a/setup.py +++ b/setup.py @@ -4,34 +4,20 @@ """ from os import path from setuptools import setup, find_packages +import json README_PATH = path.join(path.dirname(path.abspath(__file__)), 'README.rst') -with open(README_PATH, 'r') as readme: - LONG_DESCRIPTION = readme.read() +JSON_PATH = path.join(path.dirname(path.abspath(__file__)), 'setup.json') -VERSION = '1.2.1' +# Provide static information in setup.json +# such that it can be discovered automatically +with open(JSON_PATH, 'r') as info: + kwargs = json.load(info) setup( - name='reentry', - version=VERSION, - author='Rico Haeuselmann', - license='MIT License', - description='A plugin manager based on setuptools entry points mechanism', - long_description=LONG_DESCRIPTION, packages=find_packages(), include_package_data=True, package_data={'': ['js_data', 'README.rst']}, - entry_points={ - 'distutils.setup_keywords': ['reentry_register = reentry.hooks:register_dist', 'reentry_scan = reentry.hooks:scan_for_installed'], - 'console_scripts': ['reentry = reentry.cli:reentry'], - 'test_entry_points': ['test = reentry.cli:reentry'] - }, - classifiers=[ - 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', - 'Development Status :: 5 - Production/Stable', 'Environment :: Plugins', 'Intended Audience :: Developers', - 'Topic :: Software Development' - ], - install_requires=['setuptools >= 18.5', 'click', 'six', 'py'], - extras_require={ - 'dev': ['pre-commit', 'prospector', 'pylint', 'flake8', 'pytest', 'yapf', 'coverage', 'pytest-cov', 'tox', 'packaging'] - }) + long_description=open(README_PATH).read(), + long_description_content_type='text/x-rst', + **kwargs) From 36abe6176d5f178466315dd74510bb81cc43b08a Mon Sep 17 00:00:00 2001 From: Leopold Talirz Date: Wed, 17 Oct 2018 14:13:05 +0200 Subject: [PATCH 2/4] bump version number to 1.2.2 + fix to make version updater work properly with setup.json --- ops/update_version.py | 8 +++++--- reentry/__init__.py | 2 +- setup.json | 45 +++++++++++++++++++++++++++++-------------- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/ops/update_version.py b/ops/update_version.py index 2787b3f..526b0a1 100644 --- a/ops/update_version.py +++ b/ops/update_version.py @@ -57,11 +57,13 @@ def write_to_init(self): def write_to_setup(self): """Write the updated version number to setup.json.""" - with open(str(self.setup_json), 'r+') as setup_fo: + with open(str(self.setup_json), 'r') as setup_fo: # preserve order setup = json.load(setup_fo, object_pairs_hook=collections.OrderedDict) - setup['version'] = self.new_version_str - json.dump(setup, setup_fo, indent=2) + + setup['version'] = str(self.version) + with open(str(self.setup_json), 'w') as setup_fo: + json.dump(setup, setup_fo, indent=4, separators=(',', ': ')) @property def new_version_str(self): diff --git a/reentry/__init__.py b/reentry/__init__.py index 354ab0f..7411807 100644 --- a/reentry/__init__.py +++ b/reentry/__init__.py @@ -1,5 +1,5 @@ """Expose the default manager""" from reentry.default_manager import DEFAULT_MANAGER as manager -__version__ = '1.2.1' +__version__ = '1.2.2' __all__ = ['manager'] diff --git a/setup.json b/setup.json index b546bf2..54badd7 100644 --- a/setup.json +++ b/setup.json @@ -1,31 +1,48 @@ { "name": "reentry", - "version": "1.2.1", + "version": "1.2.2", "author": "Rico Haeuselmann", "license": "MIT License", "description": "A plugin manager based on setuptools entry points mechanism", "entry_points": { "distutils.setup_keywords": [ - "reentry_register = reentry.hooks:register_dist", + "reentry_register = reentry.hooks:register_dist", "reentry_scan = reentry.hooks:scan_for_installed" ], - "console_scripts": ["reentry = reentry.cli:reentry"], - "test_entry_points": ["test = reentry.cli:reentry"] + "console_scripts": [ + "reentry = reentry.cli:reentry" + ], + "test_entry_points": [ + "test = reentry.cli:reentry" + ] }, - "classifiers":[ - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 2", + "classifiers": [ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", - "Development Status :: 5 - Production/Stable", + "Development Status :: 5 - Production/Stable", "Environment :: Plugins", "Intended Audience :: Developers", "Topic :: Software Development" ], - "install_requires": ["setuptools >= 18.5", "click", "six", "py"], + "install_requires": [ + "setuptools >= 18.5", + "click", + "six", + "py" + ], "extras_require": { - "dev": ["pre-commit", "prospector==0.12.11", - "pylint", "flake8", "pytest", - "yapf", "coverage", "pytest-cov", - "tox", "packaging"] + "dev": [ + "pre-commit", + "prospector==0.12.11", + "pylint", + "flake8", + "pytest", + "yapf", + "coverage", + "pytest-cov", + "tox", + "packaging" + ] } -} +} \ No newline at end of file From fad45811825497aa3a320ac4a2b17605cb41bd7c Mon Sep 17 00:00:00 2001 From: Leopold Talirz Date: Wed, 17 Oct 2018 14:52:10 +0200 Subject: [PATCH 3/4] fix handling of json files writing setup.json back was broken when using pathlib ("must be unicode, not str") --- ops/update_version.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ops/update_version.py b/ops/update_version.py index 79f0040..dd84536 100644 --- a/ops/update_version.py +++ b/ops/update_version.py @@ -60,12 +60,12 @@ def write_to_init(self): def write_to_setup(self): """Write the updated version number to setup.json.""" - with self.setup_json.open('r') as setup_fo: + with open(str(self.setup_json), 'r') as setup_fo: # preserve order setup = json.load(setup_fo, object_pairs_hook=collections.OrderedDict) setup['version'] = str(self.version) - with self.setup_json.open('w') as setup_fo: + with open(str(self.setup_json), 'w') as setup_fo: json.dump(setup, setup_fo, indent=4, separators=(',', ': ')) @property @@ -75,7 +75,7 @@ def new_version_str(self): @property def setup_version(self): """Grab the parsed version from setup.json.""" - with self.setup_json.open('r') as setup_fo: + with open(str(self.setup_json), 'r') as setup_fo: setup = json.load(setup_fo) try: From 3ff50e77ad81a53c11669ee9e7501bca63162321 Mon Sep 17 00:00:00 2001 From: Leopold Talirz Date: Wed, 17 Oct 2018 15:29:29 +0200 Subject: [PATCH 4/4] fixes for pathlib * dependency should be pathlib2 * travis tests were still using py --- .travis-tests/host-pkg/reentry_test_host/tests.py | 13 +++++++++---- ops/update_version.py | 1 + reentry/tests/test_config.py | 2 ++ setup.json | 3 +-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.travis-tests/host-pkg/reentry_test_host/tests.py b/.travis-tests/host-pkg/reentry_test_host/tests.py index a17dda1..39482e5 100644 --- a/.travis-tests/host-pkg/reentry_test_host/tests.py +++ b/.travis-tests/host-pkg/reentry_test_host/tests.py @@ -2,7 +2,12 @@ from __future__ import print_function import click -from py import path as py_path # pylint: disable=no-name-in-module + +try: + # prefer the backport for Python <3.5 + from pathlib2 import Path +except ImportError: + from pathlib import Path from reentry import manager from reentry.config import get_datafile @@ -13,10 +18,10 @@ def main(with_noreg): """Test automatic scanning / registering""" entry_point_map = manager.get_entry_map(groups='reentry_test', ep_names=['test-plugin', 'test-noreg', 'builtin']) - data_file = py_path.local(get_datafile()) + data_file = Path(get_datafile()) assert entry_point_map, 'The test plugin entry point were not found\nMap:\n{}\n\nData File: {}\n\nContents:\n{}'.format( - manager.get_entry_map(), data_file.strpath, data_file.read()) + manager.get_entry_map(), str(data_file), data_file.read_text()) try: test_entry_point = entry_point_map['reentry_test']['test-plugin'] @@ -24,7 +29,7 @@ def main(with_noreg): if with_noreg: noreg_entry_point = entry_point_map['reentry_test']['test-noreg'] except Exception as err: - print('datafile: {}'.format(data_file.strpath)) + print('datafile: {}'.format(str(data_file))) print('\nCurrent relevant entry point map:\n\n') print(manager.format_map(entry_point_map)) print('\n') diff --git a/ops/update_version.py b/ops/update_version.py index dd84536..ff0b2af 100644 --- a/ops/update_version.py +++ b/ops/update_version.py @@ -7,6 +7,7 @@ import contextlib import subprocess try: + # prefer the backport for Python <3.5 from pathlib2 import Path except ImportError: from pathlib import Path diff --git a/reentry/tests/test_config.py b/reentry/tests/test_config.py index 27d3ac5..a18d4e6 100644 --- a/reentry/tests/test_config.py +++ b/reentry/tests/test_config.py @@ -1,9 +1,11 @@ # pylint: disable=unused-import,redefined-outer-name """Unit tests for config tools.""" try: + # prefer the backport for Python <3.5 from pathlib2 import Path except ImportError: from pathlib import Path + import pytest # pylint: disable=unused-import import six from six.moves import configparser diff --git a/setup.json b/setup.json index a035e65..dca7bdd 100644 --- a/setup.json +++ b/setup.json @@ -29,8 +29,7 @@ "setuptools >= 36.2", "click", "six", - "pathlib ; python_version<'3.5'", - "py" + "pathlib2; python_version < '3.5'" ], "extras_require": { "dev": [