Merge pull request #220 from hildogjr/master

Fix #4 (prototype on digikey)

HISTORY.rst

@@ -11,6 +11,7 @@ ______________________
 * Use the datasheet link information from KiCad and other EDAs, given by 'datasheet' field.
 * Now automatically merge 'description' and other fields to create the groups.
 * GUI save last position and size and others improvements.
+* Displa additional information from the web page distributors and use as comment in the ``cat#`` column (just implemented on DigiKey yet).
 
 
 0.1.43 (2018-03-15)

kicost/__main__.py

@@ -277,7 +277,8 @@ def main():
 
 def kicost_gui_notdependences():
     print('You don\'t have the wxPython dependence to run the GUI interface. Run once of the follow commands in terminal to install them:')
-    print('pip install wxPython # For python 2.5')
-    print('pip3 install wxPython # For python 3.0+')
-    print('Or download from last version from <https://wxpython.org/>')
+    print('pip3 install -U wxPython # For Windows & macOS')
+
+    print('pip install -U -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-16.04 wxPython # For Linux 16.04')
+    print('Or download from last version from <https://wxpython.org/pages/downloads/>')
     sys.exit(1)

kicost/distributors/__init__.py

@@ -129,6 +129,22 @@ def FakeBrowser(url):
     req.add_header('User-agent', get_user_agent())
     return req
 
+# Extra informations to by got by each part in the distributors.
+EXTRA_INFO = ['value', 'tolerance', 'footprint', 'power', 'current', 'voltage', 'frequency', 'temp_coeff', 'manf',
+              'size', 'op temp'
+              'datasheet', 'image', # Links.
+             ]
+extra_info_name_translations = {
+    'resistance': 'value',
+    'inductance': 'value',
+    'capacitance': 'value',
+    'manufacturer': 'manf',
+    'package': 'footprint',
+    'datasheets': 'datasheet',
+    'dimension': 'size',
+    'size / dimension': 'size',
+    'operating temperature': 'op temp',
+}
 
 # The global dictionary of distributor information starts out empty.
 distributor_dict = {}

kicost/distributors/digikey/digikey.py

@@ -1,6 +1,6 @@
 # MIT license
 #
-# Copyright (C) 2015 by XESS Corporation
+# Copyright (C) 2015 by XESS Corporation / Hildo Guillardi Junior
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -41,10 +41,34 @@
 from .. import urlquote, urlsplit, urlunsplit, urlopen, Request
 from .. import WEB_SCRAPE_EXCEPTIONS
 from .. import FakeBrowser
+from .. import EXTRA_INFO, extra_info_name_translations
 from ...globals import PartHtmlError
 from ...globals import logger, DEBUG_OVERVIEW, DEBUG_DETAILED, DEBUG_OBSESSIVE
 
 
+def get_extra_info(html_tree):
+    '''@brief Get the extra characteristics from the part web page.
+       @param html_tree `str()` html of the distributor part page.
+       @return `dict()` keys as characteristics names.
+    '''
+    info = {}
+    try:
+        table =  html_tree.find('table', id='prod-att-table')
+        for row in table.find_all('tr', id=None): # `None`to ignore the header row.
+            try:
+                k = row.find('th').text.strip().lower()
+                v = row.find('td').text.strip()
+                k = extra_info_name_translations.get(k, k)
+                if k in EXTRA_INFO:
+                    info[k] = v
+            except:
+                continue
+    except AttributeError:
+        # This happens when no pricing info is found in the tree.
+        logger.log(DEBUG_OBSESSIVE, 'No Digikey pricing information found!')
+    return info
+
+
 def get_price_tiers(html_tree):
     '''@brief Get the pricing tiers from the parsed tree of the Digikey product page.
        @param html_tree `str()` html of the distributor part page.
@@ -63,7 +87,6 @@ def get_price_tiers(html_tree):
     except AttributeError:
         # This happens when no pricing info is found in the tree.
         logger.log(DEBUG_OBSESSIVE, 'No Digikey pricing information found!')
-        return price_tiers  # Return empty price tiers.
     return price_tiers
 
 

kicost/distributors/web_routines.py

@@ -68,11 +68,6 @@
 
 __all__ = ['scrape_part']
 
-# Extra informations to by got by each part in the distributors.
-EXTRA_INFO = ['value', 'tolerance', 'footprint', 'power', 'current', 'voltage', 'frequency', 'temp_coeff', 'manf',
-              'datasheet', 'image' # Links.
-             ]
-
 
 def get_part_html_tree(part, dist, get_html_tree_func, local_part_html, scrape_retries, logger):
     '''Get the HTML tree for a part from the given distributor website or local HTML.'''
@@ -104,7 +99,21 @@ def get_part_html_tree(part, dist, get_html_tree_func, local_part_html, scrape_r
 
 
 def scrape_part(args):
-    '''Scrape the data for a part from each distributor website or local HTML.'''
+    '''@brief Scrape the data for a part from each distributor website or local HTML.
+    
+    Use distributors submodules to scrape each distributor part page and get
+    informations such as price, quantity avaliable and others;
+    
+    @param `int` Count of the main loop.
+    @param `str`String with the part number / distributor stock.
+    @param `dict`
+    @param `str`
+    @param `int`Number of scrape retries.
+    @param logger.getEffectiveLevel()
+    @param throttle_lock
+    @param throttle_tim
+    @return id, url, `str` distributor stock part number, `dict` price tiers, `int` qty avail, `dict` extrainfo dist
+    '''
 
     id, part, distributor_dict, local_part_html, scrape_retries, log_level, throttle_lock, throttle_timeouts = args # Unpack the arguments.
 
@@ -122,6 +131,7 @@ def scrape_part(args):
     part_num = {}
     price_tiers = {}
     qty_avail = {}
+    info_dist = {}
 
     # Scrape the part data from each distributor website or the local HTML.
     # Create a list of the distributor keys and randomly choose one of the
@@ -157,6 +167,15 @@ def scrape_part(args):
                 part_num[d] = dist_module.get_part_num(html_tree)
                 qty_avail[d] = dist_module.get_qty_avail(html_tree)
                 price_tiers[d] = dist_module.get_price_tiers(html_tree)
+
+                try:
+                    # Get extra characeristics of the part in the web page.
+                    # This will be use to comment in the 'cat#' column of the
+                    # spreadsheet and some validations (in the future implementaions)
+                    info_dist[d] = dist_module.get_extra_info(html_tree)
+                except:
+                    info_dist[d] = {}
+                    pass
 
                 # The part data has been scraped from this distributor, so remove it from the list.
                 distributors.remove(d)
@@ -167,4 +186,4 @@ def scrape_part(args):
                 throttle_lock.release()
 
     # Return the part data.
-    return id, url, part_num, price_tiers, qty_avail
+    return id, url, part_num, price_tiers, qty_avail, info_dist

kicost/kicost.py

@@ -240,11 +240,12 @@ def release(*args, **kwargs):
             for i in range(len(parts)):
                 args = (i, parts[i], distributor_dict, local_part_html, scrape_retries,
                         logger.getEffectiveLevel(), throttle_lock, throttle_timeouts)
-                id, url, part_num, price_tiers, qty_avail = scrape_part(args)
+                id, url, part_num, price_tiers, qty_avail, info_dist = scrape_part(args)
                 parts[id].part_num = part_num
                 parts[id].url = url
                 parts[id].price_tiers = price_tiers
                 parts[id].qty_avail = qty_avail
+                parts[id].info_dist = info_dist # Extra distributor web page.
                 scraping_progress.update(1)
         else:
             # Scrape data, multiple parts at a time using multiprocessing.
@@ -284,11 +285,12 @@ def update(x):
             # Get the data from each process result structure.
             logger.log(DEBUG_OVERVIEW, 'Getting the part scraped informations...')
             for result in results:
-                id, url, part_num, price_tiers, qty_avail = result.get()
+                id, url, part_num, price_tiers, qty_avail, info_dist = result.get()
                 parts[id].part_num = part_num
                 parts[id].url = url
                 parts[id].price_tiers = price_tiers
                 parts[id].qty_avail = qty_avail
+                parts[id].info_dist = info_dist # Extra distributor web page.
 
         # Done with the scraping progress bar so delete it or else we get an 
         # error when the program terminates.

kicost/spreadsheet.py

@@ -34,6 +34,7 @@
 from xlsxwriter.utility import xl_rowcol_to_cell, xl_range, xl_range_abs
 # KiCost libriries.
 from . import __version__ # Version control by @xesscorp.
+from .globals import SEPRTR
 from .globals import logger, DEBUG_OVERVIEW, DEBUG_DETAILED, DEBUG_OBSESSIVE
 from .distributors import distributor_dict # Distributors names and definitions to use in the spreadsheet.
 from .eda_tools.eda_tools import partgroup_qty, order_refs, PART_REF_REGEX
@@ -43,6 +44,9 @@
 # Regular expression to the link for one datasheet.
 DATASHEET_LINK_REGEX = re.compile('^(http(s)?:\/\/)?(www.)?[0-9a-z\.]+\/[0-9a-z\.\/\%\-\_]+(.pdf)?$', re.IGNORECASE)
 
+# Extra information characteristcs of the components gotten in the page that will be displayed as comment in the 'cat#' column.
+EXTRA_INFO_DISPLAY = ['value', 'tolerance', 'footprint', 'power', 'current', 'voltage', 'frequency', 'temp_coeff', 'manf', 'size']
+
 
 def create_spreadsheet(parts, prj_info, spreadsheet_filename, collapse_refs, user_fields, variant):
     '''Create a spreadsheet using the info for the parts (including their HTML trees).'''
@@ -757,6 +761,13 @@ def add_dist_to_worksheet(wks, wrk_formats, start_row, start_col,
             wks.write(row, start_col + columns['part_num']['col'], dist_part_num, wrk_formats['part_format'])
         else:
             dist_part_num = 'Link' # To use as text for the link.
+        try:
+            # Add a comment in the 'cat#' column with extra informations gotten in the distributor web page.
+            comment = '\n'.join(sorted([ k.capitalize()+SEPRTR+v for k, v in part.info_dist[dist].items() if k in EXTRA_INFO_DISPLAY]))
+            if comment:
+                wks.write_comment(row, start_col + columns['part_num']['col'], comment)
+        except:
+            pass
 
         # Enter a link to the distributor webpage for this part, even if there
         # is no valid quantity or pricing for the part (see next conditional).