From 482b66961efd6e18fd90891ca4fff84d3c253c00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hildo=20Guillardi=20J=C3=BAnior?= Date: Fri, 19 Jan 2018 15:22:46 -0200 Subject: [PATCH 1/3] Fixed minor bug of #102 --- kicost/eda_tools/eda_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kicost/eda_tools/eda_tools.py b/kicost/eda_tools/eda_tools.py index cee2e7141..12207f2f3 100644 --- a/kicost/eda_tools/eda_tools.py +++ b/kicost/eda_tools/eda_tools.py @@ -236,7 +236,7 @@ def group_parts(components, fields_merge): # collapsed plus `SEPRTR`. Implementation of the ISSUE #102. logger.log(DEBUG_OVERVIEW, 'Merging field asked in the identical components groups...') if fields_merge: - fields_merge = [field_name_translations.get(f.lower(), f.lower()) for f in re.split('\s', fields_merge[0])] + fields_merge = [field_name_translations.get(f.lower(), f.lower()) for f in fields_merge] for grp in new_component_groups: components_grp = dict() components_grp = {i:components[i] for i in grp.refs} From 20263ddd478565fe818ad9cd4a33b7be8bd46606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hildo=20Guillardi=20J=C3=BAnior?= Date: Fri, 19 Jan 2018 23:55:09 -0200 Subject: [PATCH 2/3] Fixed #159 improved CSV workaround KiCad Improved #159, now all file have DNP capability. Improved CSV module (some minor issue that make parts not group). Workaround KiCad issue #146, the problem is at line 153 `fields` definition. --- kicost/eda_tools/altium/altium.py | 6 +- kicost/eda_tools/csv/generic_csv.py | 8 +- kicost/eda_tools/eda_tools.py | 193 +++++++++++++++++++--------- kicost/eda_tools/kicad/kicad.py | 40 +----- kicost/kicost.py | 4 +- 5 files changed, 141 insertions(+), 110 deletions(-) diff --git a/kicost/eda_tools/altium/altium.py b/kicost/eda_tools/altium/altium.py index 9fada6ddd..ce343b1e0 100644 --- a/kicost/eda_tools/altium/altium.py +++ b/kicost/eda_tools/altium/altium.py @@ -36,7 +36,7 @@ from ...globals import logger, DEBUG_OVERVIEW, DEBUG_DETAILED, DEBUG_OBSESSIVE # Debug configurations. from ...globals import SEPRTR from ...kicost import distributor_dict -from ..eda_tools import field_name_translations, subpart_split, group_parts, split_refs +from ..eda_tools import field_name_translations, remove_dnp_parts from ..eda_tools import PART_REF_REGEX_NOT_ALLOWED # Add to deal with the fileds of Altium and WEB tools. @@ -175,6 +175,4 @@ def extract_fields_row(row, variant): 'company': None, 'date': datetime.strptime(time.ctime(os.path.getmtime(in_file)), '%a %b %d %H:%M:%S %Y').strftime("%Y-%m-%d %H:%M:%S") + ' (file)'} - #print(accepted_components) - #exit(1) - return accepted_components, prj_info + return remove_dnp_parts(accepted_components, variant), prj_info diff --git a/kicost/eda_tools/csv/generic_csv.py b/kicost/eda_tools/csv/generic_csv.py index 075f0839d..f96efa875 100644 --- a/kicost/eda_tools/csv/generic_csv.py +++ b/kicost/eda_tools/csv/generic_csv.py @@ -35,7 +35,7 @@ import re # Regular expression parser. import logging from ...globals import logger, DEBUG_OVERVIEW, DEBUG_DETAILED, DEBUG_OBSESSIVE # Debug configurations. -from ..eda_tools import field_name_translations, split_refs +from ..eda_tools import field_name_translations, remove_dnp_parts, split_refs # Add to deal with the generic CSV header purchase list. field_name_translations.update( @@ -147,7 +147,7 @@ def extract_fields(row): fields['footprint'] = vals.get('footprint', 'Foot:???') fields['value'] = vals.get('value', '???') for h in header: - if not h.lower() in ign_fields: + if not h in (ign_fields + ['refs', 'qty']): value = vals.get(h, '') if value: fields[h] = value @@ -157,7 +157,7 @@ def extract_fields(row): fields['footprint'] = vals.get('footprint', 'Foot:???').decode('utf-8') fields['value'] = vals.get('value', '???').decode('utf-8') for h in header: - if not h in ign_fields: + if not h in (ign_fields + ['refs', 'qty']): value = vals.get(h, '').decode('utf-8') if value: fields[h] = value @@ -182,4 +182,4 @@ def extract_fields(row): 'company': None, 'date': datetime.strptime(time.ctime(os.path.getmtime(in_file)), '%a %b %d %H:%M:%S %Y').strftime("%Y-%m-%d %H:%M:%S") + ' (file)'} - return accepted_components, prj_info + return remove_dnp_parts(accepted_components, variant), prj_info diff --git a/kicost/eda_tools/eda_tools.py b/kicost/eda_tools/eda_tools.py index 12207f2f3..e553d5fce 100644 --- a/kicost/eda_tools/eda_tools.py +++ b/kicost/eda_tools/eda_tools.py @@ -35,7 +35,7 @@ from ..kicost import distributor_dict from . import eda_tool_dict # EDA dictionary with the features. -__all__ = ['file_eda_match', 'subpart_qty', 'groups_sort', 'collapse_refs', 'group_parts'] +__all__ = ['file_eda_match', 'subpart_qty', 'groups_sort', 'collapse_refs', 'organize_parts'] # Qty and part separators are escaped by preceding with '\' = (?[a-z{sc}\d]*[a-z{sc}])(?P((?P\d+(\.\d+)?)({sp}(?P\d+))?)?)'.format(sc=PART_REF_REGEX_SPECIAL_CHAR_REF, sp=SUB_SEPRTR), re.IGNORECASE) # Generate a dictionary to translate all the different ways people might want -# to refer to part numbers, vendor numbers, and such. +# to refer to part numbers, vendor numbers, manufacture name and such. field_name_translations = { 'mpn': 'manf#', 'pn': 'manf#', @@ -81,25 +81,21 @@ 'part_num': 'manf#', 'p#': 'manf#', 'part#': 'manf#', + 'manf': 'manf', + 'manufacturer': 'manf', + 'mnf': 'manf', + 'man': 'manf', + 'mfg': 'manf', + 'mfr': 'manf', } - +# Crete the fields tranlate for each distributor submodule. for stub in ['part#', '#', 'p#', 'pn', 'vendor#', 'vp#', 'vpn', 'num']: for dist in distributor_dict: field_name_translations[dist + stub] = dist + '#' field_name_translations[dist + '_' + stub] = dist + '#' field_name_translations[dist + '-' + stub] = dist + '#' -field_name_translations.update( - { - 'manf': 'manf', - 'manufacturer': 'manf', - 'mnf': 'manf', - 'man': 'manf', - 'mfg': 'manf', - 'mfr': 'manf', - } -) - +# Others fileds used by KiCost and that have to be standardized. field_name_translations.update( { 'variant': 'variant', @@ -113,10 +109,11 @@ def file_eda_match(file_name): '''@brief Verify with which EDA the file matches. + + Return the EDA name with the file matches or `None` if not founded. @param file_name File `str` name. @return Name of the module correponding to read the file or `None`to not recognized. ''' - # Return the EDA name with the file matches or `None` if not founded. file_handle = open(file_name, 'r') content = file_handle.read() extension = os.path.splitext(file_name)[1] @@ -130,6 +127,26 @@ def file_eda_match(file_name): return None +def organize_parts(components, fields_merge): + '''@brief Organize the parts to better do the scrape in the distributors. + + Remove the Not Populate Parts (DNP), split the components in unique + parts, necessary because of some file formats that present the + components already grouped and to finish, group them as group parts + with same manufactures codes, company manufactures and distributors + codes to not scrape repetitively the same part kind. + + @param components Part components in a `list()` of `dict()`, format given by the EDA modules. + @return `list()` of `dict()` with the component parts organized (grouped, removed the "not populate", ...) + ''' + # Remove the Not Populate Parts. + ##components = remove_dnp_parts(components, variant) # Do this inside each EDA submodule because of the ISSUE #73. + # Split multi-components into individual subparts. + components = subpart_split(components) + # Group the components in group in the same characteristics (fields). + components = group_parts(components, fields_merge) + return components + # Temporary class for storing part group information. class IdenticalComponents(object): @@ -142,14 +159,11 @@ def group_parts(components, fields_merge): @param fileds_merge Data fields of the `dict()` variable to be merged and ignored to make the identical components group (before be scraped in the distributors web site). @return `list()` of `dict()` ''' - - # Split multi-components into individual subparts. - components = subpart_split(components) # Calculated all the fileds that never have to be used to create the hash keys. # These include all the manufacture company and codes, distributors codes # recognized by the insalled modules and, quantity and sub quantity of the part. - FIELDS_MANF = (['manf#', 'manf#_qty', 'manf', 'refs'] + [d + '#' for d in distributor_dict]) + FIELDS_MANF = (['manf#', 'manf#_qty', 'manf'] + [d + '#' for d in distributor_dict] + [d + '#_qty' for d in distributor_dict]) # Check if was asked to merge some not allowed fiels (as `manf`, `manf# ... # other ones as `desc` and even `value` and `footprint`may be merged due @@ -270,17 +284,64 @@ def group_parts(components, fields_merge): grp.fields = grp_fields # Now return the list of identical part groups. + #print('------------') + #for grp in new_component_groups: + # print(grp.refs) + #exit(1) return new_component_groups -def groups_sort(new_component_groups): +def remove_dnp_parts(components, variant): + '''@brief Remove the DNP parts or not assigned to the current variant. + + Remove components that are assigned to a variant that is not the current variant, + or which are "do not popoulate" (DNP). (Any component that does not have a variant + is assigned the current variant so it will not be removed unless it is also DNP.) + + @param components Part components in a `list()` of `dict()`, format given by the EDA modules. + @return `list()` of `dict()`. ''' - @brief Order the groups in a alphabetical way. - Put the components groups in the spreadsheet rows in a spefic order - using the reference string of the components. The order is defined - by BOM_ORDER. - @param components Part components in a `list()` of `dict()`, format given by the EDA modules. - @return Same as input. + accepted_components = {} + for ref, fields in components.items(): + # Remove DNPs. + dnp = fields.get('local:dnp', fields.get('dnp', 0)) + try: + dnp = float(dnp) + except ValueError: + pass # The field value must have been a string. + if dnp: + continue + + # Get part variant. Prioritize local variants over global ones. + variants = fields.get('local:variant', fields.get('variant', None)) + + # Remove parts that are not assigned to the current variant. + # If a part is not assigned to any variant, then it is never removed. + if variants: + # A part can be assigned to multiple variants. The part will not + # be removed if any of its variants match the current variant. + # Split the variants apart and abort the loop if any of them match. + for v in re.split('[,;/ ]', variants): + if re.match(variant, v, flags=re.IGNORECASE): + break + else: + # None of the variants matched, so skip/remove this part. + continue + + # The part was not removed, so add it to the list of accepted components. + accepted_components[ref] = fields + + return accepted_components + + +def groups_sort(new_component_groups): + '''@brief Order the groups in a alphabetical way. + + Put the components groups in the spreadsheet rows in a spefic order + using the reference string of the components. The order is defined + by BOM_ORDER. + @param components Part components in a `list()` of `dict()`, format given by the EDA modules. + @return Same as input. ''' logger.log(DEBUG_OVERVIEW, 'Sorting the groups for better visualization...') @@ -329,16 +390,17 @@ def groups_sort(new_component_groups): def subpart_split(components): - ''' - @brief Split the components with subparts in different components. - Take each part and the all manufacture/distributors combination - possibility to split in subpart the components part that have - more than one manufacture/distributors code. - For each designator... - For designator with a "single subpart" check with the quantity - is more than one. - @param components Part components in a `list()` of `dict()`, format given by the EDA modules. - @return Same as the input. + '''@brief Split the components with subparts in different components. + + Take each part and the all manufacture/distributors combination + possibility to split in subpart the components part that have + more than one manufacture/distributors code. + For each designator... + For designator with a "single subpart" check with the quantity + is more than one. + + @param components Part components in a `list()` of `dict()`, format given by the EDA modules. + @return Same as the input. ''' logger.log(DEBUG_OVERVIEW, 'Spliting subparts in the manufacture / distributors codes...') @@ -449,13 +511,14 @@ def subpart_split(components): def subpart_qty(component): - ''' - @brief Take the components quantity. - Calculate the string of the quantity of the item parsing the - referente (design) quantity and the sub quantity (in case that - was a sub part of a manufacture/distributor code). - @param components Part component `dict()`, format given by the EDA modules. - @return Quantity of the manf# part used. + '''@brief Take the components quantity. + + Calculate the string of the quantity of the item parsing the + referente (design) quantity and the sub quantity (in case that + was a sub part of a manufacture/distributor code). + + @param components Part component `dict()`, format given by the EDA modules. + @return Quantity of the manf# part used. ''' try: subqty = component.fields.get('manf#_qty') @@ -480,10 +543,12 @@ def subpart_qty(component): def subpart_list(part): ''' @brief Split the subpart by the `PART_SEPRTR`definition. + Get the list of sub parts manufacture / distributor code numbers stripping the spaces and keeping the sub part quantity information, these have to be separated by PART_SEPRTR definition. + @param part Manufacture code part `str`. @return List of manufacture code parts. ''' @@ -491,21 +556,20 @@ def subpart_list(part): def manf_code_qtypart(subpart): - ''' - @brief Get the quantity and the part code of the sub part - manufacture / distributor. Test if was pre or post - multiplied by a constant. - - Setting QTY_SEPRTR as '\:', we have - ' 4.5 : ADUM3150BRSZ-RL7' -> ('4.5', 'ADUM3150BRSZ-RL7') - '4/5 : ADUM3150BRSZ-RL7' -> ('4/5', 'ADUM3150BRSZ-RL7') - '7:ADUM3150BRSZ-RL7' -> ('7', 'ADUM3150BRSZ-RL7') - 'ADUM3150BRSZ-RL7 : 7' -> ('7', 'ADUM3150BRSZ-RL7') - 'ADUM3150BRSZ-RL7' -> ('1', 'ADUM3150BRSZ-RL7') - 'ADUM3150BRSZ-RL7:' -> ('1', 'ADUM3150BRSZ-RL7') forgot the qty understood '1' - - @param Part that way have different than ONE quantity. Intended as one element of the list of `subpart_list()`. - @return (qty, manf#) Quantity and the manufacture code. + '''@brief Get the quantity and the part code of the sub part + manufacture / distributor. Test if was pre or post + multiplied by a constant. + + Setting QTY_SEPRTR as '\:', we have + ' 4.5 : ADUM3150BRSZ-RL7' -> ('4.5', 'ADUM3150BRSZ-RL7') + '4/5 : ADUM3150BRSZ-RL7' -> ('4/5', 'ADUM3150BRSZ-RL7') + '7:ADUM3150BRSZ-RL7' -> ('7', 'ADUM3150BRSZ-RL7') + 'ADUM3150BRSZ-RL7 : 7' -> ('7', 'ADUM3150BRSZ-RL7') + 'ADUM3150BRSZ-RL7' -> ('1', 'ADUM3150BRSZ-RL7') + 'ADUM3150BRSZ-RL7:' -> ('1', 'ADUM3150BRSZ-RL7') forgot the qty understood '1' + + @param Part that way have different than ONE quantity. Intended as one element of the list of `subpart_list()`. + @return (qty, manf#) Quantity and the manufacture code. ''' subpart = re.sub(ESC_FIND, r'\1', subpart) # Remove any escape backslashes preceding PART_SEPRTR. strings = re.split(QTY_SEPRTR, subpart) @@ -548,7 +612,6 @@ def collapse_refs(refs): @param refs Designator/references `list()`. @return References in a organized view way. ''' - '''''' def convert_to_ranges(nums): # Collapse a list of numbers into sorted, comma-separated, hyphenated ranges. @@ -633,14 +696,16 @@ def to_int(n): def split_refs(text): '''@brief Split string grouped references into a unique designator. This is intended as oposite of `collapse_refs()` + + 'C17/18/19/20' --> ['C17','C18','C19','C20'] + 'C17\18\19\20' --> ['C17','C18','C19','C20'] + 'D33-D36' --> ['D33','D34','D35','D36'] + 'D33-36' --> ['D33','D34','D35','D36'] + Also ignore some caracheters as '.' or ':' used in some cases of references. + @param text Designator/references worn by a group of parts. @return Designator/references `list()` splited. ''' - # 'C17/18/19/20' --> ['C17','C18','C19','C20'] - # 'C17\18\19\20' --> ['C17','C18','C19','C20'] - # 'D33-D36' --> ['D33','D34','D35','D36'] - # 'D33-36' --> ['D33','D34','D35','D36'] - # Also ignore some caracheters as '.' or ':' used in some cases of references. partial_ref = re.split('[,;]', text) refs = [] for ref in partial_ref: diff --git a/kicost/eda_tools/kicad/kicad.py b/kicost/eda_tools/kicad/kicad.py index 89eb4636c..c742d7548 100644 --- a/kicost/eda_tools/kicad/kicad.py +++ b/kicost/eda_tools/kicad/kicad.py @@ -41,7 +41,7 @@ from ...globals import logger, DEBUG_OVERVIEW, DEBUG_DETAILED, DEBUG_OBSESSIVE from ...globals import SEPRTR from ...kicost import distributor_dict -from ..eda_tools import field_name_translations +from ..eda_tools import field_name_translations, remove_dnp_parts def get_part_groups(in_file, ignore_fields, variant): @@ -149,7 +149,8 @@ def title_find_all(data, field): # Initialize the fields from the global values in the libparts dict entry. # (These will get overwritten by any local values down below.) # (Use an empty dict if no part exists in the library.) - fields = libparts.get(libpart, dict()).copy() # Make a copy! Don't use reference! + #fields = libparts.get(libpart, dict()).copy() # Make a copy! Don't use reference! + fields = dict()#TODO # Store the part key and its value. fields['libpart'] = libpart @@ -169,37 +170,4 @@ def title_find_all(data, field): # Store the fields for the part using the reference identifier as the key. components[str(c['ref'])] = fields - # Remove components that are assigned to a variant that is not the current variant, - # or which are "do not popoulate" (DNP). (Any component that does not have a variant - # is assigned the current variant so it will not be removed unless it is also DNP.) - accepted_components = {} - for ref, fields in components.items(): - # Remove DNPs. - dnp = fields.get('local:dnp', fields.get('dnp', 0)) - try: - dnp = float(dnp) - except ValueError: - pass # The field value must have been a string. - if dnp: - continue - - # Get part variant. Prioritize local variants over global ones. - variants = fields.get('local:variant', fields.get('variant', None)) - - # Remove parts that are not assigned to the current variant. - # If a part is not assigned to any variant, then it is never removed. - if variants: - # A part can be assigned to multiple variants. The part will not - # be removed if any of its variants match the current variant. - # Split the variants apart and abort the loop if any of them match. - for v in re.split('[,;/ ]', variants): - if re.match(variant, v, flags=re.IGNORECASE): - break - else: - # None of the variants matched, so skip/remove this part. - continue - - # The part was not removed, so add it to the list of accepted components. - accepted_components[ref] = fields - - return accepted_components, prj_info + return remove_dnp_parts(components, variant), prj_info diff --git a/kicost/kicost.py b/kicost/kicost.py index 66b586145..7461871f9 100644 --- a/kicost/kicost.py +++ b/kicost/kicost.py @@ -58,7 +58,7 @@ # Import information for various EDA tools. from .eda_tools import eda_modules -from .eda_tools.eda_tools import group_parts +from .eda_tools.eda_tools import organize_parts from .spreadsheet import * # Creation of the final XLSX spreadsheet. @@ -97,7 +97,7 @@ def kicost(in_file, out_filename, user_fields, ignore_fields, group_fields, vari eda_tool_module = eda_modules[eda_tool_name[i_prj]] p, info = eda_tool_module.get_part_groups(in_file[i_prj], ignore_fields, variant[i_prj]) # Group part out of the module to merge different project lists, ignore some filed to merge, issue #131 and #102 (in the future). Next step, move the call of the function out of this loop and finish #73 implementation, remove `ignore_fields` of the call in the function above. #ISSUE. - p = group_parts(p, group_fields) + p = organize_parts(p, group_fields) # Add the project identifier in the references. for i_g in range(len(p)): p[i_g].qty = 'Board{}Qty'.format(i_prj) # 'Board{}Qty' string is used to put name quantity cells of the spreadsheet. From 31ad758578a697c61fe60b7744c74b4c2de4a45b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hildo=20Guillardi=20J=C3=BAnior?= Date: Sat, 20 Jan 2018 00:50:25 -0200 Subject: [PATCH 3/3] Fixed #97 --- kicost/eda_tools/kicad/kicad.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kicost/eda_tools/kicad/kicad.py b/kicost/eda_tools/kicad/kicad.py index c742d7548..8d7b82572 100644 --- a/kicost/eda_tools/kicad/kicad.py +++ b/kicost/eda_tools/kicad/kicad.py @@ -149,8 +149,12 @@ def title_find_all(data, field): # Initialize the fields from the global values in the libparts dict entry. # (These will get overwritten by any local values down below.) # (Use an empty dict if no part exists in the library.) - #fields = libparts.get(libpart, dict()).copy() # Make a copy! Don't use reference! - fields = dict()#TODO + fields = libparts.get(libpart, dict()).copy() # Make a copy! Don't use reference! + try: + del fields['refs'] # Delete this entry that was creating problem + # to group parts of differents sheets ISSUE #97. + except KeyError: + pass # Store the part key and its value. fields['libpart'] = libpart