Merge pull request #181 from hildogjr/master

 Something in #180 and #144

kicost/distributors/web_routines.py

@@ -79,7 +79,7 @@
 def create_local_part_html(parts, distributors):
     '''Create HTML page containing info for local (non-webscraped) parts.'''
 
-    logger.log(DEBUG_OVERVIEW, 'Creating HTML page for parts with custom pricing...')
+    logger.log(DEBUG_OVERVIEW, 'Create HTML page for parts with custom pricing...')
 
     doc, tag, text = Doc().tagtext()
     with tag('html'):
@@ -138,7 +138,6 @@ def make_random_catalog_number(p):
 
     html = doc.getvalue()
     if logger.isEnabledFor(DEBUG_OBSESSIVE):
-        print('Custom price page HTML:')
         print(indent(html))
     return html
 

kicost/eda_tools/altium/altium.py

@@ -144,21 +144,18 @@ def extract_fields_row(row, variant):
         return refs, fields
 
     # Read-in the schematic XML file to get a tree and get its root.
-    logger.log(DEBUG_OVERVIEW, 'Getting from XML Altium BoM...')
+    logger.log(DEBUG_OVERVIEW, 'Getting from XML \'{}\' Altium BoM...'.format(
+                                    os.path.basename(in_file)) )
     file_h = open(in_file)
     root = BeautifulSoup(file_h, 'lxml')
     file_h.close()
 
-    # Make a dictionary from the fields in the parts library so these field
-    # values can be instantiated into the individual components in the schematic.
-    logger.log(DEBUG_OVERVIEW, 'Getting parts library...')
-    libparts = {}
-    component_groups = {}
-
     # Get the header of the XML file of Altium, so KiCost is able to to
     # to get all the informations in the file.
+    logger.log(DEBUG_OVERVIEW, '\tGetting the XML table header...')
     header = [ extract_field(entry, 'name') for entry in root.find('columns').find_all('column') ]
 
+    logger.log(DEBUG_OVERVIEW, '\tGetting components...')
     accepted_components = {}
     for row in root.find('rows').find_all('row'):
 

kicost/eda_tools/csv/generic_csv.py

@@ -73,7 +73,8 @@ def get_part_groups(in_file, ignore_fields, variant):
 
     ign_fields = [str(f.lower()) for f in ignore_fields]
 
-    logger.log(DEBUG_OVERVIEW, 'Getting from CSV BoM...')
+    logger.log(DEBUG_OVERVIEW, 'Getting from CSV \'{}\' BoM...'.format(
+                                    os.path.basename(in_file)) )
     file_h = open(in_file)
     content = file_h.read()
     file_h.close()
@@ -91,6 +92,7 @@ def get_part_groups(in_file, ignore_fields, variant):
 
     # The first line in the file must be the column header.
     content = content.splitlines()
+    logger.log(DEBUG_OVERVIEW, '\tGetting CSV header...')
     header = next(csv.reader(content,delimiter=dialect.delimiter))
 
     # Standardize the header titles and remove the spaces before
@@ -178,7 +180,7 @@ def extract_fields(row):
 
     # Make a dictionary from the fields in the parts library so these field
     # values can be instantiated into the individual components in the schematic.
-    logger.log(DEBUG_OVERVIEW, 'Getting parts...')
+    logger.log(DEBUG_OVERVIEW, '\tGetting parts...')
 
     # Read the each line content.
     accepted_components = {}

kicost/eda_tools/eda_tools.py

@@ -176,6 +176,8 @@ def group_parts(components, fields_merge):
        @return `list()` of `dict()`
     '''
 
+    logger.log(DEBUG_OVERVIEW, 'Grouping parts...')
+
     # All codes to scrape, do not include code field name of distributors
     # that will not be scraped. This definition is used to create and check
     # the identical groups or subsplit the eemingly identical parts.
@@ -197,7 +199,7 @@ def group_parts(components, fields_merge):
     # Now partition the parts into groups of like components.
     # First, get groups of identical components but ignore any manufacturer's
     # part numbers that may be assigned. Just collect those in a list for each group.
-    logger.log(DEBUG_OVERVIEW, 'Getting groups of identical components...')
+    logger.log(DEBUG_OVERVIEW, '\tGetting groups of identical components...')
     component_groups = {}
     for ref, fields in list(components.items()): # part references and field values.
 
@@ -250,7 +252,7 @@ def group_parts(components, fields_merge):
     #       the same manf# and distributor#, even if it's `None`. It's
     #       impossible to determine which manf# the `None` parts should be
     #       assigned to, so leave their manf# as `None`.
-    logger.log(DEBUG_OVERVIEW, 'Checking the seemingly identical parts group...')
+    logger.log(DEBUG_OVERVIEW, '\tChecking the seemingly identical parts group...')
     new_component_groups = [] # Copy new component groups into this.
     for g, grp in list(component_groups.items()):
         num_manfcat_codes = {}
@@ -291,7 +293,7 @@ def group_parts(components, fields_merge):
     # so replace this field with a string composed line-by-line with the
     # ocorrences (definition `SGROUP_SEPRTR`) preceded with the refs
     # collapsed plus `SEPRTR`. Implementation of the ISSUE #102.
-    logger.log(DEBUG_OVERVIEW, 'Merging field asked in the identical components groups...')
+    logger.log(DEBUG_OVERVIEW, '\tMerging field asked in the identical components groups...')
     if fields_merge:
         fields_merge = [field_name_translations.get(f.lower(), f.lower()) for f in fields_merge]
         for grp in new_component_groups:
@@ -313,7 +315,7 @@ def group_parts(components, fields_merge):
 
     # Now get the values of all fields within the members of a group.
     # These will become the field values for ALL members of that group.
-    logger.log(DEBUG_OVERVIEW, 'Propagating field values to identical components...')
+    logger.log(DEBUG_OVERVIEW, '\tPropagating field values to identical components...')
     for grp in new_component_groups:
         grp_fields = {}
         qty = []
@@ -350,12 +352,15 @@ def remove_dnp_parts(components, variant):
     '''@brief Remove the DNP parts or not assigned to the current variant.
        
        Remove components that are assigned to a variant that is not the current variant,
-       or which are "do not popoulate" (DNP). (Any component that does not have a variant
+       or which are "do not populate" (DNP). (Any component that does not have a variant
        is assigned the current variant so it will not be removed unless it is also DNP.)
        
        @param components Part components in a `list()` of `dict()`, format given by the EDA modules.
        @return `list()` of `dict()`.
     '''
+
+    logger.log(DEBUG_OVERVIEW, '\tRemoving do not populate parts...')
+
     accepted_components = {}
     for ref, fields in components.items():
         # Remove DNPs.

kicost/eda_tools/kicad/kicad.py

@@ -98,12 +98,14 @@ def extract_fields(part, variant):
         return fields
 
     # Read-in the schematic XML file to get a tree and get its root.
-    logger.log(DEBUG_OVERVIEW, 'Getting from XML KiCad BoM...')
+    logger.log(DEBUG_OVERVIEW, 'Getting from XML \'{}\' KiCad BoM...'.format(
+                                    os.path.basename(in_file)) )
     file_h = open(in_file)
     root = BeautifulSoup(file_h, 'lxml')
     file_h.close()
 
     # Get the general information of the project BoM XML file.
+    logger.log(DEBUG_OVERVIEW, '\tGetting authorship data...')
     title = root.find('title_block')
     def title_find_all(data, field):
         '''Helper function for finding title info, especially if it is absent.'''
@@ -118,7 +120,7 @@ def title_find_all(data, field):
 
     # Make a dictionary from the fields in the parts library so these field
     # values can be instantiated into the individual components in the schematic.
-    logger.log(DEBUG_OVERVIEW, 'Getting parts library...')
+    logger.log(DEBUG_OVERVIEW, '\tGetting parts library...')
     libparts = {}
     if root.find('libparts'):
         for p in root.find('libparts').find_all('libpart'):
@@ -140,7 +142,7 @@ def title_find_all(data, field):
     # Find the components used in the schematic and elaborate
     # them with global values from the libraries and local values
     # from the schematic.
-    logger.log(DEBUG_OVERVIEW, 'Get components...')
+    logger.log(DEBUG_OVERVIEW, '\tGetting components...')
     components = {}
     for c in root.find('components').find_all('comp'):
 

kicost/kicost.py

@@ -203,16 +203,19 @@ def update(x):
             return x
 
         # Start the web scraping processes, one for each part.
+        logger.log(DEBUG_OVERVIEW, 'Starting {} parallels process...'.format(num_processes))
         results = [pool.apply_async(scrape_part, [args], callback=update) for args in arg_sets]
 
         # Wait for all the processes to have results, then kill-off all the scraping processes.
         for r in results:
             while(not r.ready()):
                 pass
+        logger.log(DEBUG_OVERVIEW, 'All parallels process finished with success.')
         pool.close()
         pool.join()
 
         # Get the data from each process result structure.
+        logger.log(DEBUG_OVERVIEW, 'Getting the part scraped informations...')
         for result in results:
             id, url, part_num, price_tiers, qty_avail = result.get()
             parts[id].part_num = part_num

kicost/kicost_gui.py

@@ -32,7 +32,7 @@
 except ImportError:
     raise ImportError('wxPython package not recognised.')
 import webbrowser # To update informations.
-import os, subprocess # To access OS commands and run in the shell.
+import sys, os, subprocess # To access OS commands and run in the shell.
 import platform # To check the system platform when open the XLS file.
 import tempfile # To create the temporary log file.
 from datetime import datetime # To create the log name, when asked to save.
@@ -59,6 +59,21 @@
 #https://github.com/xesscorp/KiCost/blob/master/kicost/version.py
 
 
+
+def open_file(filepath):
+    '''@brief Open a file with the default application by yht different OS.
+       @param filepath str() file name.
+    '''
+    if sys.platform.startswith('darwin'): # Mac-OS.
+        subprocess.call(('open', filepath))
+    elif sys.platform.startswith('windows'): # Windows.
+        os.startfile(filepath)
+    elif sys.platform.startswith('linux'): # Linux.
+        subprocess.call(('xdg-open', filepath))
+    else:
+        print('Not recognized OS.')
+
+
 class FileDropTarget( wx.FileDropTarget ):
     ''' This object implements Drop Target functionality for Files.
         @param Window handle.
@@ -84,7 +99,9 @@ def __init__( self, parent ):
 
         mmi = wx.MenuItem(self, wx.NewId(), '&Purge')
         self.Append(mmi)
-        self.Bind(wx.EVT_MENU, self.clearMessages, mmi)
+        self.Bind(wx.EVT_MENU, self.purgeMessages, mmi)
+
+        self.AppendSeparator()
 
         mmi = wx.MenuItem(self, wx.NewId(), '&Copy to clipboard')
         self.Append(mmi)
@@ -94,9 +111,14 @@ def __init__( self, parent ):
         self.Append(mmi)
         self.Bind(wx.EVT_MENU, self.saveMessages, mmi)
 
+        mmi = wx.MenuItem(self, wx.NewId(), 'S&ave and clear')
+        self.Append(mmi)
+        self.Bind(wx.EVT_MENU, self.saveClearMessages, mmi)
+
         mmi = wx.MenuItem(self, wx.NewId(), '&Open externally')
         self.Append(mmi)
         self.Bind(wx.EVT_MENU, self.openMessages, mmi)
+
 
     def copyMessages( self, event ):
         ''' @brief Copy the warning/error/log messages to clipboard. '''
@@ -107,10 +129,10 @@ def copyMessages( self, event ):
         wx.TheClipboard.SetData(clipdata)
         wx.TheClipboard.Close()
 
-    def clearMessages( self, event ):
+    def purgeMessages( self, event ):
         ''' @brief Clear message box. '''
         event.Skip()
-        self.parent.m_textCtrlMessages.SetValue('')
+        self.parent.m_textCtrlMessages.Clear()
 
     def saveMessages( self, event ):
         ''' @brief Save the messages as a text "KiCost*.log" file. '''
@@ -132,20 +154,19 @@ def saveMessages( self, event ):
             wx.MessageBox('The log file as saved.', 'Info', wx.OK | wx.ICON_INFORMATION)
         dlg.Destroy()
 
+    def saveClearMessages( self, event ):
+        '''@brief Save the messages and clear the log in the guide.'''
+        self.saveMessages(event)
+        self.purgeMessages(event)
+
     def openMessages( self, event ):
         ''' @brief Save the messages in a temporary file and open it in the default text editor before sytem deletation. '''
         event.Skip()
-        self.parent.m_textCtrlMessages.SetValue('This is just test message')
-        with tempfile.NamedTemporaryFile(prefix='KiCost_', suffix='.log', delete=True, mode='w+t') as temp:
+
+        self.parent.m_textCtrlMessages.AppendText('\naqui\nhjhk')
+        with tempfile.NamedTemporaryFile(prefix='KiCost_', suffix='.log', delete=True, mode='w') as temp:
             temp.write( self.parent.m_textCtrlMessages.GetValue() )
-            if platform.system()=='Linux':
-                os.system( 'xdg-open ' + '"' + temp.name + '"' )
-            elif platform.system()=='Windows':
-                os.system( 'start ' + '"' + temp.name + '"' )
-            elif platform.system()=='Darwin': # Mac-OS
-                os.system( 'open -n ' + '"' + temp.name + '"' )
-            else:
-                print('Not recognized OS.')
+            open_file(temp.name)
             temp.close()
 
 
@@ -448,11 +469,11 @@ def m_comboBox_files_selecthist( self, event):
             self.m_comboBox_files.Insert( fileNames, 0 )
             self.updateEDAselection() # Auto-select the EDA module.
         else:
-            self.m_comboBox_files.SetValue( '' )
+            self.m_comboBox_files.SetValue('')
 
     #----------------------------------------------------------------------
     def updateEDAselection( self ):
-        ''' @brief Update the EDA selection in the listBox based on the comboBox actual text '''
+        ''' @brief Update the EDA selection in the listBox based on the comboBox actual text. '''
         fileNames = re.split(SEP_FILES, self.m_comboBox_files.GetValue())
         if len(fileNames)==1:
             eda_module = file_eda_match(fileNames[0])
@@ -542,15 +563,87 @@ def button_run( self, event ):
     #----------------------------------------------------------------------
     def run( self ):
         ''' @brief Run KiCost in the GUI interface updating the process bar and messages. '''
-    #TODO
-    # Messages and process bar on the GUI without CLI, remove the `runTerminal` call here.
-    #TODO `runTerminal`
-    # Keep this for `--user` parameter, if passed aditional ones, overwrite the saved to execute KiCost.
         self.m_gauge_process.SetValue(0)
+        self.m_textCtrlMessages.Clear()
+
+        class argments:
+            pass
+        args = argments()
+
+        args.input = re.split(SEP_FILES, self.m_comboBox_files.GetValue())
+
+        spreadsheet_file = re.split(SEP_FILES, self.m_comboBox_files.GetValue())
+        if len(spreadsheet_file)==1:
+            spreadsheet_file = os.path.splitext( spreadsheet_file[0] )[0] + '.xlsx'
+        else:
+            spreadsheet_file = output_filename_multipleinputs( spreadsheet_file )
+        # Handle case where output is going into an existing spreadsheet file.
+        if os.path.isfile(spreadsheet_file):
+            if not self.m_checkBox_overwrite.GetValue():
+                dlg = wx.MessageDialog(self, 
+                    "The file output \'{}\' already exit, do you wnat overwrite?".format(
+                                os.path.basename(spreadsheet_file)
+                            ),
+                    "Confirm Overwrite", wx.YES_NO|wx.YES_DEFAULT|wx.ICON_QUESTION|wx.STAY_ON_TOP|wx.CENTER)
+                result = dlg.ShowModal()
+                dlg.Destroy()
+                if result==wx.ID_NO:
+                    self.m_textCtrlMessages.AppendText('\nNot able to overwrite \'{}\'...'.format(
+                                os.path.basename(spreadsheet_file)
+                            )
+                        )
+                    return
+        args.output = spreadsheet_file
+
+        if self.m_textCtrlextracmd.GetValue():
+            extra_commands = ' ' + self.m_textCtrlextracmd.GetValue()
+        else:
+            extra_commands = []
+        args.fields = ''.join( re.findall('--fields (.+)', extra_commands) or re.findall('-f (.+)', extra_commands) ).split()
+        args.ignore_fields = ''.join( re.findall('--ignore_fields (.+)', extra_commands) or re.findall('-ign (.+)', extra_commands) ).split()
+        args.group_fields = ''.join( re.findall('--group_fields (.+)', extra_commands) or re.findall('-grp (.+)', extra_commands) ).split()
+        args.variant = ''.join( re.findall('--variant (.+)', extra_commands) or re.findall('-var (.+)', extra_commands) )
+
+        num_processes = self.m_spinCtrl_np.GetValue() # Parallels process scrapping.
+        args.retries = self.m_spinCtrl_retries.GetValue() # Retry time in the scraps.
+        args.throttling_delay = self.m_spinCtrlDouble_throttling.GetValue() # Delay between consecutive scrapes.
+
+        if self.m_listBox_edatool.GetStringSelection():
+            for k,v in eda_tool_dict.items():
+               if v['label']==self.m_listBox_edatool.GetStringSelection():
+                  eda_module = v['module']
+                  break
+        args.eda_tool = eda_module
+
+        # Get the current distributors to scrape.
+        choisen_dist = list(self.m_checkList_dist.GetCheckedItems())
+        if choisen_dist:
+            dist_list = []
+            #choisen_dist = [self.m_checkList_dist.GetString(idx) for idx in choisen_dist]
+            for idx in choisen_dist:
+                label = self.m_checkList_dist.GetString(idx)
+                for k,v in distributor_dict.items():
+                    if v['label']==label:
+                        dist_list.append( v['module'] )
+                        break
+        args.include = dist_list
+        args.exclude = []
+
+        kicost(in_file=args.input, out_filename=args.output,
+            user_fields=args.fields, ignore_fields=args.ignore_fields, group_fields=args.group_fields,
+            variant=args.variant, num_processes=num_processes, eda_tool_name=args.eda_tool,
+            exclude_dist_list=args.exclude, include_dist_list=args.include,
+            scrape_retries=args.retries, throttling_delay=args.throttling_delay)
 
-        self.runTerminal()
+        self.m_gauge_process.SetValue(100)
+
+        if self.m_checkBox_openXLS.GetValue():
+            self.m_textCtrlMessages.AppendText('\nOpening the output file \'{}\'...'.format(
+                                os.path.basename(spreadsheet_file)
+                            )
+                        )
+            open_file(spreadsheet_file)
 
-        self.m_gauge_process.SetValue(50)
         return
 
     #----------------------------------------------------------------------