From 5a904941bbfea34beb29d4fa167b7c14581779ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hildo=20Guillardi=20J=C3=BAnior?= Date: Mon, 4 Jun 2018 17:27:15 +0200 Subject: [PATCH] TME fix and clean-up imports --- AUTHORS.rst | 3 ++- HISTORY.rst | 12 +++++++----- kicost/distributors/digikey/digikey.py | 12 ++---------- kicost/distributors/farnell/farnell.py | 15 +++------------ kicost/distributors/local/local.py | 6 +----- kicost/distributors/mouser/mouser.py | 15 +++------------ kicost/distributors/newark/newark.py | 15 +++------------ kicost/distributors/rs/rs.py | 7 +------ kicost/distributors/tme/tme.py | 18 ++++-------------- 9 files changed, 26 insertions(+), 77 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index df70686fb..8e7c4299a 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -16,4 +16,5 @@ Contributors * Diorcet Yann: https://github.com/diorcety * Giacinto Luigi Cerone https://github.com/glcerone * Hildo Guillardi Júnior https://github.com/hildogjr -* Adam Heinrich https://github.com/adamheinrich \ No newline at end of file +* Adam Heinrich https://github.com/adamheinrich +* Max Maisel https://github.com/mmmaisel diff --git a/HISTORY.rst b/HISTORY.rst index f8ca7a592..a9165c305 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,14 +7,16 @@ History ______________________ * Changed Farnell link and table result format. - +* Fixed TME `fake_browser` migration. +* Re-factored the distributors modules to class kind and improved the scrape sequence to decrease probability of ban. +* Fixed the multi-threading configuration. 0.1.44 (2018-05-28) ______________________ -* Fixed `logging` messages when using `tqdm`(process bar) for sequencial scrape, missing fix for multithreads scrape. -* Improve the `spreadsheet.py` to a lighter file when use just one distributor. -* Improved log messages to better comunity debug. +* Fixed ``logging`` messages when using ``tqdm``(process bar) for sequential scrape, missing fix for multithreads scrape. +* Improve the ``spreadsheet.py`` to a lighter file when use just one distributor. +* Improved log messages to better community debug. * Add Upverter CSV compatibility. * Fixed Mouser "quote price" exception in the price tiers. * Fixed wxPython exception import. @@ -38,7 +40,7 @@ ______________________ * Added ``--group_fields`` option to ignore differences in fields of the components and group them. * Fixed the not ungrouping issue when ``manf#`` equal ``None``. * CSV now accepts files from Proteus and Eagle EDA tools. -* Cleared up unused Python imports and better placed functions into files (spreasheet creation files are now in ``spreadsheet.py``). +* Cleared up unused Python imports and better placed functions into files (spreadsheet creation files are now in ``spreadsheet.py``). * Added a KiCost stamp version at the end of the spreadsheet and file information in the beginning, if they are not inside it. * Fixed issues related to user visualization in the spreadsheet (added gray formatted conditioning and the "exclude desc and manf columns"). * Added "user errors" and software scape in the case of not recognized references characters given the message of how to solve. diff --git a/kicost/distributors/digikey/digikey.py b/kicost/distributors/digikey/digikey.py index 2c8e5ca87..2dc56c7d0 100644 --- a/kicost/distributors/digikey/digikey.py +++ b/kicost/distributors/digikey/digikey.py @@ -21,23 +21,15 @@ # THE SOFTWARE. # Inserted by Pasteurize tool. -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import division -from __future__ import absolute_import -from builtins import zip -from builtins import range -from builtins import int -from builtins import str +from __future__ import print_function, unicode_literals, division, absolute_import +from builtins import zip, range, int, str from future import standard_library standard_library.install_aliases() - import future import re, difflib from bs4 import BeautifulSoup import http.client # For web scraping exceptions. -#from .. import urlencode, urlquote, urlsplit, urlunsplit from .. import fake_browser from .. import EXTRA_INFO_DIST, extra_info_dist_name_translations from ...globals import PartHtmlError diff --git a/kicost/distributors/farnell/farnell.py b/kicost/distributors/farnell/farnell.py index 3d0afdce5..fa64308f9 100644 --- a/kicost/distributors/farnell/farnell.py +++ b/kicost/distributors/farnell/farnell.py @@ -21,24 +21,15 @@ # THE SOFTWARE. # Inserted by Pasteurize tool. -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import division -from __future__ import absolute_import -from builtins import zip -from builtins import range -from builtins import int -from builtins import str +from __future__ import print_function, unicode_literals, division, absolute_import +from builtins import zip, range, int, str from future import standard_library standard_library.install_aliases() - import future -import re -import difflib +import re, difflib from bs4 import BeautifulSoup import http.client # For web scraping exceptions. -#from .. import urlencode, urlquote, urlsplit, urlunsplit from .. import fake_browser from ...globals import PartHtmlError from ...globals import currency diff --git a/kicost/distributors/local/local.py b/kicost/distributors/local/local.py index a5ced0487..93fff0723 100644 --- a/kicost/distributors/local/local.py +++ b/kicost/distributors/local/local.py @@ -21,14 +21,10 @@ # THE SOFTWARE. # Inserted by Pasteurize tool. -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import division -from __future__ import absolute_import +from __future__ import print_function, unicode_literals, division, absolute_import from builtins import zip, range, int, str from future import standard_library standard_library.install_aliases() - import future import re, difflib diff --git a/kicost/distributors/mouser/mouser.py b/kicost/distributors/mouser/mouser.py index be2108267..ea724f78a 100644 --- a/kicost/distributors/mouser/mouser.py +++ b/kicost/distributors/mouser/mouser.py @@ -21,24 +21,15 @@ # THE SOFTWARE. # Inserted by Pasteurize tool. -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import division -from __future__ import absolute_import -from builtins import zip -from builtins import range -from builtins import int -from builtins import str +from __future__ import print_function, unicode_literals, division, absolute_import +from builtins import zip, range, int, str from future import standard_library standard_library.install_aliases() - import future -import re -import difflib +import re, difflib from bs4 import BeautifulSoup import http.client # For web scraping exceptions. -#from .. import urlencode, urlquote, urlsplit, urlunsplit from .. import fake_browser from ...globals import PartHtmlError from ...globals import logger, DEBUG_OVERVIEW, DEBUG_DETAILED, DEBUG_OBSESSIVE, DEBUG_HTTP_RESPONSES diff --git a/kicost/distributors/newark/newark.py b/kicost/distributors/newark/newark.py index 5f216b9e1..5deb4a5f6 100644 --- a/kicost/distributors/newark/newark.py +++ b/kicost/distributors/newark/newark.py @@ -21,24 +21,15 @@ # THE SOFTWARE. # Inserted by Pasteurize tool. -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import division -from __future__ import absolute_import -from builtins import zip -from builtins import range -from builtins import int -from builtins import str +from __future__ import print_function, unicode_literals, division, absolute_import +from builtins import zip, range, int, str from future import standard_library standard_library.install_aliases() - import future -import re -import difflib +import re, difflib from bs4 import BeautifulSoup import http.client # For web scraping exceptions. -#from .. import urlencode, urlquote, urlsplit, urlunsplit from .. import fake_browser from ...globals import PartHtmlError from ...globals import logger, DEBUG_OVERVIEW, DEBUG_DETAILED, DEBUG_OBSESSIVE, DEBUG_HTTP_RESPONSES diff --git a/kicost/distributors/rs/rs.py b/kicost/distributors/rs/rs.py index 1fd87af0e..5ca7dbc5b 100644 --- a/kicost/distributors/rs/rs.py +++ b/kicost/distributors/rs/rs.py @@ -21,20 +21,15 @@ # THE SOFTWARE. # Inserted by Pasteurize tool. -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import division -from __future__ import absolute_import +from __future__ import print_function, unicode_literals, division, absolute_import from builtins import zip, range, int, str from future import standard_library standard_library.install_aliases() - import future import re, difflib from bs4 import BeautifulSoup import http.client # For web scraping exceptions. -#from .. import urlencode, urlquote, urlsplit, urlunsplit from .. import fake_browser from ...globals import PartHtmlError from ...globals import logger, DEBUG_OVERVIEW, DEBUG_DETAILED, DEBUG_OBSESSIVE, DEBUG_HTTP_RESPONSES diff --git a/kicost/distributors/tme/tme.py b/kicost/distributors/tme/tme.py index 1c22b6d89..138771d96 100644 --- a/kicost/distributors/tme/tme.py +++ b/kicost/distributors/tme/tme.py @@ -21,25 +21,16 @@ # THE SOFTWARE. # Inserted by Pasteurize tool. -from __future__ import print_function -from __future__ import unicode_literals -from __future__ import division -from __future__ import absolute_import -from builtins import zip -from builtins import range -from builtins import int -from builtins import str +from __future__ import print_function, unicode_literals, division, absolute_import +from builtins import zip, range, int, str from future import standard_library standard_library.install_aliases() - import future -import re -import difflib +import re, difflib import json from bs4 import BeautifulSoup import http.client # For web scraping exceptions. -#from .. import urlencode, urlquote, urlsplit, urlunsplit from .. import fake_browser from ...globals import PartHtmlError from ...globals import logger, DEBUG_OVERVIEW, DEBUG_DETAILED, DEBUG_OBSESSIVE, DEBUG_HTTP_RESPONSES @@ -73,8 +64,7 @@ def __ajax_details(self, pn): return None, None try: - r = r.decode('utf-8') # Convert bytes to string in Python 3. - p = json.loads(r).get('Products') + p = json.loads(html).get('Products') if p is not None and isinstance(p, list): p = p[0] html_tree = BeautifulSoup(p.get('PriceTpl', '').replace("\n", ""), "lxml")