From 0ba6f998c8e0120b4c8aa95713956d4babb455d8 Mon Sep 17 00:00:00 2001 From: Dick Hollenbeck Date: Sun, 1 Sep 2013 12:49:01 -0500 Subject: [PATCH] Clarify, extend, and rename ky*.py to kicad_netlist_reader.py. Significantly enhance the bom_csv_grouped_by_value.py BOM generator. IMO it at this moment, the best BOM production tool for KiCad. --- scripts/bom-in-python/bom-generation.py | 30 +- scripts/bom-in-python/bom_csv_by_ref.py | 14 +- scripts/bom-in-python/bom_csv_by_ref_v2.py | 12 +- .../bom-in-python/bom_csv_grouped_by_value.py | 102 ++- scripts/bom-in-python/bom_html_by_value.py | 28 +- .../bom_html_grouped_by_value.py | 24 +- scripts/bom-in-python/kicad_netlist_reader.py | 727 ++++++++++++++++++ .../ky_generic_netlist_reader.py | 450 ----------- scripts/bom-in-python/round_robin.py | 12 +- scripts/bom-in-python/round_value_robin.py | 12 +- 10 files changed, 897 insertions(+), 514 deletions(-) create mode 100644 scripts/bom-in-python/kicad_netlist_reader.py delete mode 100644 scripts/bom-in-python/ky_generic_netlist_reader.py diff --git a/scripts/bom-in-python/bom-generation.py b/scripts/bom-in-python/bom-generation.py index 8746575348..c0cda72fd8 100644 --- a/scripts/bom-in-python/bom-generation.py +++ b/scripts/bom-in-python/bom-generation.py @@ -4,8 +4,10 @@ # Example: Sorted and Grouped HTML BOM with more advanced grouping # +from __future__ import print_function + # Import the KiCad python helper module and the csv formatter -import ky_generic_netlist_reader +import kicad_netlist_reader import sys # Start with a basic html template @@ -43,9 +45,9 @@ def myEqu(self, other): result = True if self.getValue() != other.getValue(): result = False - elif self.getLib() != other.getLib(): + elif self.getLibName() != other.getLibName(): result = False - elif self.getPart() != other.getPart(): + elif self.getPartName() != other.getPartName(): result = False elif self.getFootprint() != other.getFootprint(): result = False @@ -61,18 +63,18 @@ def myEqu(self, other): # Override the component equivalence operator - it is important to do this # before loading the netlist, otherwise all components will have the original # equivalency operator. -ky_generic_netlist_reader.component.__equ__ = myEqu +kicad_netlist_reader.comp.__equ__ = myEqu # Generate an instance of a generic netlist, and load the netlist tree from # video.tmp. If the file doesn't exist, execution will stop -net = ky_generic_netlist_reader.netlist(sys.argv[1]) +net = kicad_netlist_reader.netlist(sys.argv[1]) # Open a file to write too, if the file cannot be opened output to stdout # instead try: f = open(sys.argv[2], 'w') except IOError: - print >> sys.stderr, __file__, ":", e + print(__file__, ":", e, file=sys.stderr) f = stdout # Output a set of rows for a header providing general information @@ -88,9 +90,11 @@ row += "Description" + "Vendor" html = html.replace('', row + "") +components = net.getInterestingComponents() + # Get all of the components in groups of matching parts + values -# (see ky_generic_netlist_reader.py) -grouped = net.groupComponents() +# (see kicad_netlist_reader.py) +grouped = net.groupComponents(components) # Output all of the component information for group in grouped: @@ -99,17 +103,19 @@ for group in grouped: # Add the reference of every component in the group and keep a reference # to the component so that the other data can be filled in once per group for component in group: - refs += component.getRef() + ", " + if len(refs) > 0: + refs += ", " + refs += component.getRef() c = component row = "\n " row += "" + refs +"" + str(len(group)) - row += "" + c.getValue() + "" + c.getLib() + "/" - row += c.getPart() + "" + c.getDatasheet() + "" + row += "" + c.getValue() + "" + c.getLibName() + ":" + row += c.getPartName() + "" + c.getDatasheet() + "" row += c.getDescription() + "" + c.getField("Vendor") row += "" html = html.replace('', row + "") # Print the formatted html to output file -print >> f, html +print(html, file=f) diff --git a/scripts/bom-in-python/bom_csv_by_ref.py b/scripts/bom-in-python/bom_csv_by_ref.py index ccc874fd16..51f8e34a39 100644 --- a/scripts/bom-in-python/bom_csv_by_ref.py +++ b/scripts/bom-in-python/bom_csv_by_ref.py @@ -4,21 +4,23 @@ # Example: Tab delimited list (The same as std output) Ungrouped # +from __future__ import print_function + # Import the KiCad python helper module and the csv formatter -import ky_generic_netlist_reader +import kicad_netlist_reader import csv import sys # Generate an instance of a generic netlist, and load the netlist tree from # the command line option. If the file doesn't exist, execution will stop -net = ky_generic_netlist_reader.netlist(sys.argv[1]) +net = kicad_netlist_reader.netlist(sys.argv[1]) # Open a file to write to, if the file cannot be opened output to stdout # instead try: f = open(sys.argv[2], 'w') except IOError: - print >> sys.stderr, __file__, ":", e + print(__file__, ":", e, file=sys.stderr) f = stdout # Create a new csv writer object to use as the output formatter, although we @@ -32,7 +34,9 @@ out.writerow(['Tool:', net.getTool()]) out.writerow(['Component Count:', len(net.components)]) out.writerow(['Ref', 'Value', 'Part', 'Documentation', 'Description', 'Vendor']) +components = net.getInterestingComponents() + # Output all of the component information -for c in net.components: - out.writerow([c.getRef(), c.getValue(), c.getLib() + "/" + c.getPart(), +for c in components: + out.writerow([c.getRef(), c.getValue(), c.getLibName() + ":" + c.getPartName(), c.getDatasheet(), c.getDescription(), c.getField("Vendor")]) diff --git a/scripts/bom-in-python/bom_csv_by_ref_v2.py b/scripts/bom-in-python/bom_csv_by_ref_v2.py index 38e6f0c75f..e3bef12118 100644 --- a/scripts/bom-in-python/bom_csv_by_ref_v2.py +++ b/scripts/bom-in-python/bom_csv_by_ref_v2.py @@ -4,21 +4,23 @@ # Example: Ungrouped (One component per row) CSV output # +from __future__ import print_function + # Import the KiCad python helper module -import ky_generic_netlist_reader +import kicad_netlist_reader import csv import sys # Generate an instance of a generic netlist, and load the netlist tree from # the command line option. If the file doesn't exist, execution will stop -net = ky_generic_netlist_reader.netlist(sys.argv[1]) +net = kicad_netlist_reader.netlist(sys.argv[1]) # Open a file to write to, if the file cannot be opened output to stdout # instead try: f = open(sys.argv[2], 'w') except IOError: - print >> sys.stderr, __file__, ":", e + print(__file__, ":", e, file=sys.stderr) f = stdout # Create a new csv writer object to use as the output formatter @@ -31,8 +33,10 @@ out.writerow(['Tool:', net.getTool()]) out.writerow(['Component Count:', len(net.components)]) out.writerow(['Ref', 'Value', 'Footprint', 'Datasheet', 'Manufacturer', 'Vendor']) +components = net.getInterestingComponents() + # Output all of the component information (One component per row) -for c in net.components: +for c in components: out.writerow([c.getRef(), c.getValue(), c.getFootprint(), c.getDatasheet(), c.getField("Manufacturer"), c.getField("Vendor")]) diff --git a/scripts/bom-in-python/bom_csv_grouped_by_value.py b/scripts/bom-in-python/bom_csv_grouped_by_value.py index 81d3acecf6..dba801bc06 100644 --- a/scripts/bom-in-python/bom_csv_grouped_by_value.py +++ b/scripts/bom-in-python/bom_csv_grouped_by_value.py @@ -4,49 +4,123 @@ # Example: Sorted and Grouped CSV BOM # +from __future__ import print_function + # Import the KiCad python helper module and the csv formatter -import ky_generic_netlist_reader +import kicad_netlist_reader import csv import sys + +if len(sys.argv) != 3: + print("Usage ", __file__, " ", file=sys.stderr) + sys.exit(1) + + # Generate an instance of a generic netlist, and load the netlist tree from # the command line option. If the file doesn't exist, execution will stop -net = ky_generic_netlist_reader.netlist(sys.argv[1]) +net = kicad_netlist_reader.netlist(sys.argv[1]) # Open a file to write to, if the file cannot be opened output to stdout # instead try: f = open(sys.argv[2], 'w') except IOError: - print >> sys.stderr, __file__, ":", e + print(__file__, ":", e, file=sys.stderr) f = stdout -# Create a new csv writer object to use as the output formatter -out = csv.writer(f, lineterminator='\n', delimiter=',', quotechar='\"', quoting=csv.QUOTE_ALL) +# subset the components to those wanted in the BOM, controlled +# by block in kicad_netlist_reader.py +components = net.getInterestingComponents() -# Output a set of rows for a header providing general information +compfields = net.gatherComponentFieldUnion(components) +partfields = net.gatherLibPartFieldUnion() + +# remove Reference, Value, Datasheet, and Footprint, they will come from 'columns' below +partfields -= set( ['Reference', 'Value', 'Datasheet', 'Footprint'] ) + +columnset = compfields | partfields # union + +# prepend an initial 'hard coded' list and put the enchillada into list 'columns' +columns = ['Item', 'Qty', 'Reference(s)', 'Value', 'LibPart', 'Footprint', 'Datasheet'] + sorted(list(columnset)) + +# Create a new csv writer object to use as the output formatter +out = csv.writer(f, lineterminator='\n', delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL) + +# Output a set of rows as a header providing general information out.writerow(['Source:', net.getSource()]) out.writerow(['Date:', net.getDate()]) out.writerow(['Tool:', net.getTool()]) -out.writerow(['Component Count:', len(net.components)]) -out.writerow(['Ref', 'Qnty', 'Value', 'Part', 'Datasheet', 'Description', 'Vendor']) +out.writerow(['Component Count:', len(components)]) +out.writerow([]) +out.writerow(['Individual Components:']) +out.writerow([]) # blank line +out.writerow(columns) + +# Output all the interesting components individually first: +row = [] +for c in components: + del row[:] + row.append('') # item is blank in individual table + row.append('') # Qty is always 1, why print it + row.append( c.getRef() ) # Reference + row.append( c.getValue() ) # Value + row.append( c.getLibName() + ":" + c.getPartName() ) # LibPart + #row.append( c.getDescription() ) + row.append( c.getFootprint() ) + row.append( c.getDatasheet() ) + + # from column 7 upwards, use the fieldnames to grab the data + for field in columns[7:]: + row.append( c.getField( field ) ); + + out.writerow(row) + + +out.writerow([]) # blank line +out.writerow([]) # blank line +out.writerow([]) # blank line + +out.writerow(['Collated Components:']) +out.writerow([]) # blank line +out.writerow(columns) # reuse same columns + + # Get all of the components in groups of matching parts + values -# (see ky_generic_netlist_reader.py) -grouped = net.groupComponents() +# (see kicad_netlist_reader.py) +grouped = net.groupComponents(components) -# Output all of the component information + +# Output component information organized by group, aka as collated: +item = 0 for group in grouped: + del row[:] refs = "" # Add the reference of every component in the group and keep a reference # to the component so that the other data can be filled in once per group for component in group: - refs += component.getRef() + ", " + if len(refs) > 0: + refs += ", " + refs += component.getRef() c = component # Fill in the component groups common data - out.writerow([refs, len(group), c.getValue(), c.getLib() + "/" + c.getPart(), c.getDatasheet(), - c.getDescription(), c.getField("Vendor")]) + # columns = ['Item', 'Qty', 'Reference(s)', 'Value', 'LibPart', 'Footprint', 'Datasheet'] + sorted(list(columnset)) + item += 1 + row.append( item ) + row.append( len(group) ) + row.append( refs ); + row.append( c.getValue() ) + row.append( c.getLibName() + ":" + c.getPartName() ) + row.append( net.getGroupFootprint(group) ) + row.append( net.getGroupDatasheet(group) ) + # from column 7 upwards, use the fieldnames to grab the data + for field in columns[7:]: + row.append( net.getGroupField(group, field) ); + out.writerow( row ) + +f.close() diff --git a/scripts/bom-in-python/bom_html_by_value.py b/scripts/bom-in-python/bom_html_by_value.py index 4c7fa5f2a5..244248ad71 100644 --- a/scripts/bom-in-python/bom_html_by_value.py +++ b/scripts/bom-in-python/bom_html_by_value.py @@ -4,8 +4,10 @@ # Example: Sorted and Grouped HTML BOM with more advanced grouping # +from __future__ import print_function + # Import the KiCad python helper module and the csv formatter -import ky_generic_netlist_reader +import kicad_netlist_reader import sys # Start with a basic html template @@ -43,9 +45,9 @@ def myEqu(self, other): result = True if self.getValue() != other.getValue(): result = False - elif self.getLib() != other.getLib(): + elif self.getLibName() != other.getLibName(): result = False - elif self.getPart() != other.getPart(): + elif self.getPartName() != other.getPartName(): result = False elif self.getFootprint() != other.getFootprint(): result = False @@ -61,18 +63,18 @@ def myEqu(self, other): # Override the component equivalence operator - it is important to do this # before loading the netlist, otherwise all components will have the original # equivalency operator. -ky_generic_netlist_reader.component.__equ__ = myEqu +kicad_netlist_reader.comp.__equ__ = myEqu # Generate an instance of a generic netlist, and load the netlist tree from # video.xml. If the file doesn't exist, execution will stop -net = ky_generic_netlist_reader.netlist(sys.argv[1]) +net = kicad_netlist_reader.netlist(sys.argv[1]) # Open a file to write too, if the file cannot be opened output to stdout # instead try: f = open(sys.argv[2], 'w') except IOError: - print >> sys.stderr, __file__, ":", e + print(__file__, ":", e, file=sys.stderr) f = stdout # Output a set of rows for a header providing general information @@ -90,9 +92,11 @@ row += "PartNumber" + "Vendor" html = html.replace('', row + "") +components = net.getInterestingComponents() + # Get all of the components in groups of matching parts + values -# (see ky_generic_netlist_reader.py) -grouped = net.groupComponents() +# (see kicad_netlist_reader.py) +grouped = net.groupComponents(components) # Output all of the component information for group in grouped: @@ -101,12 +105,14 @@ for group in grouped: # Add the reference of every component in the group and keep a reference # to the component so that the other data can be filled in once per group for component in group: - refs += component.getRef() + ", " + if len(refs) > 0: + refs += ", " + refs += component.getRef() c = component row = "" + refs +"" + str(len(group)) row += "" + c.getValue() + "" - row += c.getLib() + "/" + c.getPart() + "" + row += c.getLibName() + ":" + c.getPartName() + "" #row += c.getDatasheet() + "" row += c.getDescription() + "" row += c.getField("PartNumber") + "" @@ -116,4 +122,4 @@ for group in grouped: html = html.replace('', row + "") # Print the formatted html to output file -print >> f, html +print(html, file=f) diff --git a/scripts/bom-in-python/bom_html_grouped_by_value.py b/scripts/bom-in-python/bom_html_grouped_by_value.py index d1cb8912f7..11bcc5dfb0 100644 --- a/scripts/bom-in-python/bom_html_grouped_by_value.py +++ b/scripts/bom-in-python/bom_html_grouped_by_value.py @@ -4,8 +4,10 @@ # Example: Sorted and Grouped HTML BOM # +from __future__ import print_function + # Import the KiCad python helper module and the csv formatter -import ky_generic_netlist_reader +import kicad_netlist_reader import sys # Start with a basic html template @@ -30,14 +32,14 @@ html = """ # Generate an instance of a generic netlist, and load the netlist tree from # the command line option. If the file doesn't exist, execution will stop -net = ky_generic_netlist_reader.netlist(sys.argv[1]) +net = kicad_netlist_reader.netlist(sys.argv[1]) # Open a file to write to, if the file cannot be opened output to stdout # instead try: f = open(sys.argv[2], 'w') except IOError: - print >> sys.stderr, __file__, ":", e + print(__file__, ":", e, file=sys.stderr) f = stdout # Output a set of rows for a header providing general information @@ -53,9 +55,11 @@ row += "Description" + "Vendor" html = html.replace('', row + "") +components = net.getInterestingComponents() + # Get all of the components in groups of matching parts + values -# (see ky_generic_netlist_reader.py) -grouped = net.groupComponents() +# (see kicad_netlist_reader.py) +grouped = net.groupComponents(components) # Output all of the component information for group in grouped: @@ -64,16 +68,18 @@ for group in grouped: # Add the reference of every component in the group and keep a reference # to the component so that the other data can be filled in once per group for component in group: - refs += component.getRef() + ", " + if len(refs) > 0: + refs += ", " + refs += component.getRef() c = component row = "" + refs +"" + str(len(group)) - row += "" + c.getValue() + "" + c.getLib() + "/" - row += c.getPart() + "" + c.getDatasheet() + "" + row += "" + c.getValue() + "" + c.getLibName() + ":" + row += c.getPartName() + "" + c.getDatasheet() + "" row += c.getDescription() + "" + c.getField("Vendor") row += "" html = html.replace('', row + "") # Print the formatted html to the file -print >> f, html +print(html, file=f) diff --git a/scripts/bom-in-python/kicad_netlist_reader.py b/scripts/bom-in-python/kicad_netlist_reader.py new file mode 100644 index 0000000000..8a01adcd19 --- /dev/null +++ b/scripts/bom-in-python/kicad_netlist_reader.py @@ -0,0 +1,727 @@ +# +# KiCad python module for interpreting generic netlists which can be used +# to generate Bills of materials, etc. +# +# No string formatting is used on purpose as the only string formatting that +# is current compatible with python 2.4+ to 3.0+ is the '%' method, and that +# is due to be deprecated in 3.0+ soon +# + +from __future__ import print_function +import sys +import xml.sax as sax +import re +import pdb + +#--------------------------------------------------------------------- + +# excluded_fields is a list of regular expressions. If any one matches a field +# from either a component or a libpart, then that will not be included as a +# column in the BOM. Otherwise all columns from all used libparts and components +# will be unionized and will appear. Some fields are impossible to blacklist, such +# as Ref, Value, Footprint, and Datasheet. Additionally Qty and Item are supplied +# unconditionally as columns, and may not be removed. +excluded_fields = [ + #'Price@1000' + ] + + +# You may exlude components from the BOM by either: +# +# 1) adding a custom field named "Installed" to your components and filling it +# with a value of "NU" (Normally Uninstalled). +# See netlist.getInterestingComponents(), or +# +# 2) blacklisting it in any of the three following lists: + + +# regular expressions which match component 'Reference' fields of components that +# are to be excluded from the BOM. +excluded_references = [ + 'TP[0-9]+' # all test points + ] + + +# regular expressions which match component 'Value' fields of components that +# are to be excluded from the BOM. +excluded_values = [ + 'MOUNTHOLE', + 'SCOPETEST', + 'MOUNT_HOLE', + 'SOLDER_BRIDGE.*' + ] + + +# regular expressions which match component 'Footprint' fields of components that +# are to be excluded from the BOM. +excluded_footprints = [ + #'MOUNTHOLE' + ] + +#-------------------------------------------------------------------- + + +class xmlElement(): + """xml element which can represent all nodes of the netlist tree. It can be + used to easily generate various output formats by propogating format + requests to children recursively. + """ + def __init__(self, name, parent=None): + self.name = name + self.attributes = {} + self.parent = parent + self.chars = "" + self.children = [] + + def __str__(self): + """String representation of this netlist element + + """ + return self.name + "[" + self.chars + "]" + " attr_count:" + str(len(self.attributes)) + + def formatXML(self, nestLevel=0, amChild=False): + """Return this element formatted as XML + + Keywords: + nestLevel -- increases by one for each level of nesting. + amChild -- If set to True, the start of document is not returned. + + """ + s = "" + + indent = "" + for i in range(nestLevel): + indent += " " + + if not amChild: + s = "\n" + + s += indent + "<" + self.name + for a in self.attributes: + s += " " + a + "=\"" + self.attributes[a] + "\"" + + if (len(self.chars) == 0) and (len(self.children) == 0): + s += "/>" + else: + s += ">" + self.chars + + for c in self.children: + s += "\n" + s += c.formatXML(nestLevel+1, True) + + if (len(self.children) > 0): + s += "\n" + indent + + if (len(self.children) > 0) or (len(self.chars) > 0): + s += "" + + return s + + def formatHTML(self, amChild=False): + """Return this element formatted as HTML + + Keywords: + amChild -- If set to True, the start of document is not returned + + """ + s = "" + + if not amChild: + s = """ + + + + + + + + """ + + s += "\n" + + for c in self.children: + s += c.formatHTML(True) + + if not amChild: + s += """
" + self.name + "
" + self.chars + "
    " + for a in self.attributes: + s += "
  • " + a + " = " + self.attributes[a] + "
    • " + + s += "
+ + """ + + return s + + def addAttribute(self, attr, value): + """Add an attribute to this element""" + self.attributes[attr] = value + + def setAttribute(self, attr, value): + """Set an attributes value - in fact does the same thing as add + attribute + + """ + self.attributes[attr] = value + + def setChars(self, chars): + """Set the characters for this element""" + self.chars = chars + + def addChars(self, chars): + """Add characters (textual value) to this element""" + self.chars += chars + + def addChild(self, child): + """Add a child element to this element""" + self.children.append(child) + return self.children[len(self.children) - 1] + + def getParent(self): + """Get the parent of this element (Could be None)""" + return self.parent + + def getChild(self, name): + """Returns the first child element named 'name' + + Keywords: + name -- The name of the child element to return""" + for child in self.children: + if child.name == name: + return child + return None + + def getChildren(self, name=None): + if name: + # return _all_ children named "name" + ret = [] + for child in self.children: + if child.name == name: + ret.append(child) + return ret + else: + return self.children + + def get(self, elemName, attribute="", attrmatch=""): + """Return the text data for either an attribute or an xmlElement + """ + if (self.name == elemName): + if attribute != "": + try: + if attrmatch != "": + if self.attributes[attribute] == attrmatch: + return self.chars + else: + return self.attributes[attribute] + except AttributeError: + return "" + else: + return self.chars + + for child in self.children: + ret = child.get(elemName, attribute, attrmatch) + if ret != "": + return ret + + return "" + + + +class libpart(): + """Class for a library part, aka 'libpart' in the xml netlist file. + (Components in eeschema are instantiated from library parts.) + This part class is implemented by wrapping an xmlElement with accessors. + This xmlElement instance is held in field 'element'. + """ + def __init__(self, xml_element): + # + self.element = xml_element + + #def __str__(self): + # simply print the xmlElement associated with this part + #return str(self.element) + + def getLibName(self): + return self.element.get("libpart", "lib") + + def getPartName(self): + return self.element.get("libpart", "part") + + def getDescription(self): + return self.element.get("description") + + def getField(self, name): + return self.element.get("field", "name", name) + + def getFieldNames(self): + """Return a list of field names in play for this libpart. + """ + fieldNames = [] + fields = self.element.getChild('fields') + if fields: + for f in fields.getChildren(): + fieldNames.append( f.get('field','name') ) + return fieldNames + + def getDatasheet(self): + return self.getField("Datasheet") + + def getFootprint(self): + return self.getField("Footprint") + + def getAliases(self): + """Return a list of aliases or None""" + aliases = self.element.getChild("aliases") + if aliases: + ret = [] + children = aliases.getChildren() + # grab the text out of each child: + for child in children: + ret.append( child.get("alias") ) + return ret + return None + + +class comp(): + """Class for a component, aka 'comp' in the xml netlist file. + This component class is implemented by wrapping an xmlElement instance + with accessors. The xmlElement is held in field 'element'. + """ + + def __init__(self, xml_element): + self.element = xml_element + self.libpart = None + + # Set to true when this component is included in a component group + self.grouped = False + + def __eq__(self, other): + """Equlivalency operator, remember this can be easily overloaded""" + result = False + if self.getValue() == other.getValue(): + if self.getLibName() == other.getLibName(): + if self.getPartName() == other.getPartName(): + result = True + return result + + def setLibPart(self, part): + self.libpart = part + + def getLibPart(self): + return self.libpart + + def getPartName(self): + return self.element.get("libsource", "part") + + def getLibName(self): + return self.element.get("libsource", "lib") + + def setValue(self, value): + """Set the value of this component""" + v = self.element.getChild("value") + if v: + v.setChars(value) + + def getValue(self): + return self.element.get("value") + + def getField(self, name, libraryToo=True): + """Return the value of a field named name. The component is first + checked for the field, and then the components library part is checked + for the field. If the field doesn't exist in either, an empty string is + returned + + Keywords: + name -- The name of the field to return the value for + libraryToo -- look in the libpart's fields for the same name if not found + in component itself + """ + + field = self.element.get("field", "name", name) + if field == "" and libraryToo: + field = self.libpart.getField(name) + return field + + def getFieldNames(self): + """Return a list of field names in play for this component. Mandatory + fields are not included, and they are: Value, Footprint, Datasheet, Ref. + The netlist format only includes fields with non-empty values. So if a field + is empty, it will not be present in the returned list. + """ + fieldNames = [] + fields = self.element.getChild('fields') + if fields: + for f in fields.getChildren(): + fieldNames.append( f.get('field','name') ) + return fieldNames + + def getRef(self): + return self.element.get("comp", "ref") + + def getFootprint(self, libraryToo=True): + ret = self.element.get("footprint") + if ret =="" and libraryToo: + ret = self.libpart.getFootprint() + return ret + + def getDatasheet(self, libraryToo=True): + ret = self.element.get("datasheet") + if ret == '' and libraryToo: + ret = self.libpart.getDatasheet() + return ret + + def getTimestamp(self): + return self.element.get("tstamp") + + def getDescription(self): + return self.libpart.getDescription() + + +class netlist(): + """ Kicad generic netlist class. Generally loaded from a kicad generic + netlist file. Includes several helper functions to ease BOM creating + scripts + + """ + def __init__(self, fname=""): + """Initialiser for the genericNetlist class + + Keywords: + fname -- The name of the generic netlist file to open (Optional) + + """ + self.design = None + self.components = [] + self.libparts = [] + self.libraries = [] + self.nets = [] + + # The entire tree is loaded into self.tree + self.tree = [] + + self._curr_element = None + + # component blacklist regexs, made from exluded_* above. + self.excluded_references = [] + self.excluded_values = [] + self.excluded_footprints = [] + + if fname != "": + self.load(fname) + + def addChars(self, content): + """Add characters to the current element""" + self._curr_element.addChars(content) + + def addElement(self, name): + """Add a new kicad generic element to the list""" + if self._curr_element == None: + self.tree = xmlElement(name) + self._curr_element = self.tree + else: + self._curr_element = self._curr_element.addChild( + xmlElement(name, self._curr_element)) + + # If this element is a component, add it to the components list + if self._curr_element.name == "comp": + self.components.append(comp(self._curr_element)) + + # Assign the design element + if self._curr_element.name == "design": + self.design = self._curr_element + + # If this element is a library part, add it to the parts list + if self._curr_element.name == "libpart": + self.libparts.append(libpart(self._curr_element)) + + # If this element is a net, add it to the nets list + if self._curr_element.name == "net": + self.nets.append(self._curr_element) + + # If this element is a library, add it to the libraries list + if self._curr_element.name == "library": + self.libraries.append(self._curr_element) + + return self._curr_element + + def endDocument(self): + """Called when the netlist document has been fully parsed""" + # When the document is complete, the library parts must be linked to + # the components as they are seperate in the tree so as not to + # duplicate library part information for every component + for c in self.components: + for p in self.libparts: + if p.getLibName() == c.getLibName(): + if p.getPartName() == c.getPartName(): + c.setLibPart(p) + break + else: + aliases = p.getAliases() + if aliases and self.aliasMatch( c.getPartName(), aliases ): + c.setLibPart(p) + break; + + if not c.getLibPart(): + print( 'missing libpart for ref:', c.getRef(), c.getPartName(), c.getLibName() ) + + + def aliasMatch(self, partName, aliasList): + for alias in aliasList: + if partName == alias: + return True + return False + + def endElement(self): + """End the current element and switch to its parent""" + self._curr_element = self._curr_element.getParent() + + def getDate(self): + """Return the date + time string generated by the tree creation tool""" + return self.design.get("date") + + def getSource(self): + """Return the source string for the design""" + return self.design.get("source") + + def getTool(self): + """Return the tool string which was used to create the netlist tree""" + return self.design.get("tool") + + def gatherComponentFieldUnion(self, components=None): + """Gather the complete 'set' of unique component fields, fields found in any component. + """ + if not components: + components=self.components + + s = set() + for c in components: + s.update( c.getFieldNames() ) + + # omit anything matching any regex in excluded_fields + ret = set() + for field in s: + exclude = False + for rex in excluded_fields: + if re.match( rex, field ): + exclude = True + break + if not exclude: + ret.add(field) + + return ret # this is a python 'set' + + def gatherLibPartFieldUnion(self): + """Gather the complete 'set' of part fields, fields found in any part. + """ + s = set() + for p in self.libparts: + s.update( p.getFieldNames() ) + + # omit anything matching any regex in excluded_fields + ret = set() + for field in s: + exclude = False + for rex in excluded_fields: + if re.match( rex, field ): + exclude = True + break + if not exclude: + ret.add(field) + + return ret # this is a python 'set' + + def getInterestingComponents(self): + """Return a subset of all components, those that should show up in the BOM. + Omit those that should not, by consulting the blacklists: + excluded_values, excluded_refs, and excluded_footprints, which hold one + or more regular expressions. If any of the the regular expressions match + the corresponding field's value in a component, then the component is exluded. + """ + + # pre-compile all the regex expressions: + del self.excluded_references[:] + del self.excluded_values[:] + del self.excluded_footprints[:] + + for rex in excluded_references: + self.excluded_references.append( re.compile( rex ) ) + + for rex in excluded_values: + self.excluded_values.append( re.compile( rex ) ) + + for rex in excluded_footprints: + self.excluded_footprints.append( re.compile( rex ) ) + + # the subset of components to return, considered as "interesting". + ret = [] + + # run each component thru a series of tests, if it passes all, then add it + # to the interesting list 'ret'. + for c in self.components: + exclude = False + if not exclude: + for refs in self.excluded_references: + if refs.match(c.getRef()): + exclude = True + break; + if not exclude: + for vals in self.excluded_values: + if vals.match(c.getValue()): + exclude = True + break; + if not exclude: + for mods in self.excluded_footprints: + if mods.match(c.getFootprint()): + exclude = True + break; + + if not exclude: + # This is a fairly personal way to flag DNS (Do Not Stuff). NU for + # me means Normally Uninstalled. You can 'or in' another expression here. + if c.getField( "Installed" ) == 'NU': + exclude = True + + if not exclude: + ret.append(c) + + # Sort first by ref as this makes for easier to read BOM's + ret.sort(key=lambda g: g.getRef()) + + return ret + + + def groupComponents(self, components = None): + """Return a list of component lists. Components are grouped together + when the value, library and part identifiers match. + + Keywords: + components -- is a list of components, typically an interesting subset + of all components, or None. If None, then all components are looked at. + """ + if not components: + components = self.components + + groups = [] + + # Make sure to start off will all components ungrouped to begin with + for c in components: + c.grouped = False + + # Group components based on the value, library and part identifiers + for c in components: + if c.grouped == False: + c.grouped = True + newgroup = [] + newgroup.append(c) + + # Check every other ungrouped component against this component + # and add to the group as necessary + for ci in components: + if ci.grouped == False and ci == c: + newgroup.append(ci) + ci.grouped = True + + # Add the new component group to the groups list + groups.append(newgroup) + + # Each group is a list of components, we need to sort each list first + # to get them in order as this makes for easier to read BOM's + for g in groups: + g = sorted(g, key=lambda g: g.getRef()) + + # Finally, sort the groups to order the references alphabetically + groups = sorted(groups, key=lambda group: group[0].getRef()) + + return groups + + def getGroupField(self, group, field): + """Return the whatever is known about the given field by consulting each + component in the group. If any of them know something about the property/field, + then return that first non-blank value. + """ + for c in group: + ret = c.getField(field, False) + if ret != '': + return ret + return group[0].getLibPart().getField(field) + + def getGroupFootprint(self, group): + """Return the whatever is known about the Footprint by consulting each + component in the group. If any of them know something about the Footprint, + then return that first non-blank value. + """ + for c in group: + ret = c.getFootprint() + if ret != "": + return ret + return group[0].getLibPart().getFootprint() + + def getGroupDatasheet(self, group): + """Return the whatever is known about the Datasheet by consulting each + component in the group. If any of them know something about the Datasheet, + then return that first non-blank value. + """ + for c in group: + ret = c.getDatasheet() + if ret != "": + return ret + + if len(group) > 0: + return group[0].getLibPart().getDatasheet() + else: + print("NULL!") + return '' + + def formatXML(self): + """Return the whole netlist formatted in XML""" + return self.tree.formatXML() + + def formatHTML(self): + """Return the whole netlist formatted in HTML""" + return self.tree.formatHTML() + + def load(self, fname): + """Load a kicad generic netlist + + Keywords: + fname -- The name of the generic netlist file to open + + """ + try: + self._reader = sax.make_parser() + self._reader.setContentHandler(_gNetReader(self)) + self._reader.parse(fname) + except IOError as e: + print( __file__, ":", e, file=sys.stderr ) + sys.exit(-1) + + + +class _gNetReader(sax.handler.ContentHandler): + """SAX kicad generic netlist content handler - passes most of the work back + to the 'netlist' class which builds a complete tree in RAM for the design + + """ + def __init__(self, aParent): + self.parent = aParent + + def startElement(self, name, attrs): + """Start of a new XML element event""" + element = self.parent.addElement(name) + + for name in attrs.getNames(): + element.addAttribute(name, attrs.getValue(name)) + + def endElement(self, name): + self.parent.endElement() + + def characters(self, content): + # Ignore erroneous white space - ignoreableWhitespace does not get rid + # of the need for this! + if not content.isspace(): + self.parent.addChars(content) + + def endDocument(self): + """End of the XML document event""" + self.parent.endDocument() diff --git a/scripts/bom-in-python/ky_generic_netlist_reader.py b/scripts/bom-in-python/ky_generic_netlist_reader.py deleted file mode 100644 index 03fcaf3d73..0000000000 --- a/scripts/bom-in-python/ky_generic_netlist_reader.py +++ /dev/null @@ -1,450 +0,0 @@ -# -# KiCad python module for interpreting generic netlists which can be used -# to generate Bills of materials, etc. -# -# No string formatting is used on purpose as the only string formatting that -# is current compatible with python 2.4+ to 3.0+ is the '%' method, and that -# is due to be deprecated in 3.0+ soon -# - -import sys -import xml.sax as sax - - -class component(): - """Class for a set of component information""" - def __init__(self, element): - self.element = element - self.libpart = None - - # Set to true when this component is included in a component group - self.grouped = False - - def __eq__(self, other): - """Equlivalency operator, remember this can be easily overloaded""" - result = False - if self.getValue() == other.getValue(): - if self.getLib() == other.getLib(): - if self.getPart() == other.getPart(): - result = True - return result - - def setPart(self, part): - self.libpart = part - - def setValue(self, value): - """Set the value of this component""" - v = self.element.getChild("value") - if v: - v.setChars(value) - - def getValue(self): - return self.element.get("value") - - def getRef(self): - return self.element.get("comp", "ref") - - def getFootprint(self): - return self.element.get("footprint") - - def getDatasheet(self): - return self.element.get("datasheet") - - def getLib(self): - return self.element.get("libsource", "lib") - - def getPart(self): - return self.element.get("libsource", "part") - - def getTimestamp(self): - return self.element.get("tstamp") - - def getDescription(self): - # When attempting to access the part, we must take care in case the part - # cannot be found in the netlist - try: - d = self.libpart.getDescription() - except AttributeError: - d = "" - return d - - def getDatasheet(self): - # When attempting to access the part, we must take care in case the part - # cannot be found in the netlist - try: - d = self.libpart.getDatasheet() - except AttributeError: - d = "" - return d - - def getField(self, name): - """Return the value of a field named name. The component is first - checked for the field, and then the components library part is checked - for the field. If the field doesn't exist in either, an empty string is - returned - - Keywords: - name -- The name of the field to return the value for - - """ - field = self.element.get("field", "name", name) - if field == "": - try: - field = self.libpart.getField(name) - except AttributeError: - field = "" - return field - - -class netlistElement(): - """Generic netlist element. All elements for a netlist tree which can be - used to easily generate various output formats by propogating format - requests to all children - """ - def __init__(self, name, parent=None): - self.name = name - self.attributes = {} - self.parent = parent - self.chars = "" - self.children = [] - self.indent = "" - - def __str__(self): - """String representation of this netlist element - - """ - return (self.name + "[" + self.chars + "]" + " attr:" + - str(len(self.attributes[a]))) - - def formatXML(self, amChild=False): - """Return this element formatted as XML - - Keywords: - amChild -- If set to True, the start of document is not returned - - """ - s = "" - - if not amChild: - s = "\n" - - s += self.indent + "<" + self.name - for a in self.attributes: - s += " " + a + "=\"" + self.attributes[a] + "\"" - - if (len(self.chars) == 0) and (len(self.children) == 0): - s += "/>" - else: - s += ">" + self.chars - - for c in self.children: - c.indent += self.indent + " " - s += "\n" - s += c.formatXML(True) - - if (len(self.children) > 0): - s += "\n" + self.indent - - if (len(self.children) > 0) or (len(self.chars) > 0): - s += "" - - return s - - def formatHTML(self, amChild=False): - """Return this element formatted as HTML - - Keywords: - amChild -- If set to True, the start of document is not returned - - """ - s = "" - - if not amChild: - s = """ - - - - - - - - """ - - s += "\n" - - for c in self.children: - s += c.formatHTML(True) - - if not amChild: - s += """
" + self.name + "
" + self.chars + "
    " - for a in self.attributes: - s += "
  • " + a + " = " + self.attributes[a] + "
    • " - - s += "
- - """ - - return s - - def addAttribute(self, attr, value): - """Add an attribute to this element""" - self.attributes[attr] = value - - def setChars(self, chars): - """Set the characters for this element""" - self.chars = chars - - def addChars(self, chars): - """Add characters (textual value) to this element""" - self.chars += chars - - def addChild(self, child): - """Add a child element to this element""" - self.children.append(child) - return self.children[len(self.children) - 1] - - def getParent(self): - """Get the parent of this element (Could be None)""" - return self.parent - - def setAttribute(self, attr, value): - """Set an attributes value - in fact does the same thing as add - attribute - - """ - self.attributes[attr] = value - - def getChild(self, name): - """Returns a child element of name - - Keywords: - name -- The name of the child element to return""" - for child in self.children: - if child.name == name: - return child - - return None - - def get(self, element, attribute="", attrmatch=""): - """Return the data for either an attribute, or else an element""" - if (self.name == element): - if attribute != "": - if attrmatch != "": - if self.attributes[attribute] == attrmatch: - return self.chars - else: - return self.attributes[attribute] - else: - return self.chars - - for child in self.children: - if child.get(element, attribute, attrmatch) != "": - return child.get(element, attribute, attrmatch) - - return "" - - -class netlist(): - """ Kicad generic netlist class. Generally loaded from a kicad generic - netlist file. Includes several helper functions to ease BOM creating - scripts - - """ - def __init__(self, fname=""): - """Initialiser for the genericNetlist class - - Keywords: - fname -- The name of the generic netlist file to open (Optional) - - """ - self.design = None - self.components = [] - self.libparts = [] - self.libraries = [] - self.nets = [] - - # The entire tree is loaded into self.tree - self.tree = [] - - self._curr_element = None - - if fname != "": - self.load(fname) - - def addChars(self, content): - """Add characters to the current element""" - self._curr_element.addChars(content) - - def addElement(self, name): - """Add a new kicad generic element to the list""" - if self._curr_element == None: - self.tree = netlistElement(name) - self._curr_element = self.tree - else: - self._curr_element = self._curr_element.addChild( - netlistElement(name, self._curr_element)) - - # If this element is a component, add it to the components list - if self._curr_element.name == "comp": - self.components.append(component(self._curr_element)) - - # Assign the design element - if self._curr_element.name == "design": - self.design = self._curr_element - - # If this element is a library part, add it to the parts list - if self._curr_element.name == "libpart": - self.libparts.append(part(self._curr_element)) - - # If this element is a net, add it to the nets list - if self._curr_element.name == "net": - self.nets.append(self._curr_element) - - # If this element is a library, add it to the libraries list - if self._curr_element.name == "library": - self.libraries.append(self._curr_element) - - return self._curr_element - - def endDocument(self): - """Called when the netlist document has been fully parsed""" - # When the document is complete, the library parts must be linked to - # the components as they are seperate in the tree so as not to - # duplicate library part information for every component - for c in self.components: - for p in self.libparts: - if p.getPart() == c.getPart() and p.getLib() == c.getLib(): - c.setPart(p) - - def endElement(self): - """End the current element and switch to its parent""" - self._curr_element = self._curr_element.getParent() - - def getDate(self): - """Return the date + time string generated by the tree creation tool""" - return self.design.get("date") - - def getSource(self): - """Return the source string for the design""" - return self.design.get("source") - - def getTool(self): - """Return the tool string which was used to create the netlist tree""" - return self.design.get("tool") - - def groupComponents(self): - """Return a list of component lists. Components are grouped together - when the value, library and part identifiers match - - """ - groups = [] - - # Make sure to start off will all components ungrouped to begin with - for c in self.components: - c.grouped = False - - # Group components based on the value, library and part identifiers - for c in self.components: - if c.grouped == False: - c.grouped = True - newgroup = [] - newgroup.append(c) - - # Check every other ungrouped component against this component - # and add to the group as necessary - for ci in self.components: - if ci.grouped == False and ci == c: - newgroup.append(ci) - ci.grouped = True - - # Add the new component group to the groups list - groups.append(newgroup) - - # Each group is a list of components, we need to sort each list first - # to get them in order as this makes for easier to read BOM's - for g in groups: - g = sorted(g, key=lambda g: g.getRef()) - - # Finally, sort the groups to order the references alphabetically - groups = sorted(groups, key=lambda group: group[0].getRef()) - - return groups - - def formatXML(self): - """Return the whole netlist formatted in XML""" - return self.tree.formatXML() - - def formatHTML(self): - """Return the whole netlist formatted in HTML""" - return self.tree.formatHTML() - - def load(self, fname): - """Load a kicad generic netlist - - Keywords: - fname -- The name of the generic netlist file to open - - """ - try: - self._reader = sax.make_parser() - self._reader.setContentHandler(_gNetReader(self)) - self._reader.parse(fname) - except IOError as e: - print >> sys.stderr, __file__, ":", e - sys.exit(-1) - - -class part(): - """Class for a library part""" - def __init__(self, part): - # The part is a reference to a libpart generic netlist element - self.element = part - - def __str__(self): - # simply print the generic netlist element associated with this part - return str(self.element) - - def getDatasheet(self): - return self.element.get("docs") - - def getLib(self): - return self.element.get("libpart", "lib") - - def getPart(self): - return self.element.get("libpart", "part") - - def getDescription(self): - return self.element.get("description") - - def getField(self, name): - return self.element.get("field", "name", name) - - -class _gNetReader(sax.handler.ContentHandler): - """SAX kicad generic netlist content handler - passes most of the work back - to the gNetlist class which builds a complete tree in RAM for the design - - """ - def __init__(self, aParent): - self.parent = aParent - - def startElement(self, name, attrs): - """Start of a new XML element event""" - element = self.parent.addElement(name) - - for name in attrs.getNames(): - element.addAttribute(name, attrs.getValue(name)) - - def endElement(self, name): - self.parent.endElement() - - def characters(self, content): - # Ignore erroneous white space - ignoreableWhitespace does not get rid - # of the need for this! - if not content.isspace(): - self.parent.addChars(content) - - def endDocument(self): - """End of the XML document event""" - self.parent.endDocument() diff --git a/scripts/bom-in-python/round_robin.py b/scripts/bom-in-python/round_robin.py index 6aff7ef1a9..245f7ad16d 100644 --- a/scripts/bom-in-python/round_robin.py +++ b/scripts/bom-in-python/round_robin.py @@ -4,20 +4,24 @@ # Example: Round robin, XML to XML conversion # +from __future__ import print_function + # Import the KiCad python helper module and the csv formatter -import ky_generic_netlist_reader +import kicad_netlist_reader import sys +import pdb + # Generate an instance of a generic netlist, and load the netlist tree from # the command line option. If the file doesn't exist, execution will stop -net = ky_generic_netlist_reader.netlist(sys.argv[1]) +net = kicad_netlist_reader.netlist(sys.argv[1]) # Open a file to write to, if the file cannot be opened output to stdout # instead try: f = open(sys.argv[2], 'w') except IOError: - print >> sys.stderr, __file__, ":", e + print( __file__, ":", e, file=sys.stderr) f = stdout -print >> f, net.formatXML() +print(net.formatXML(), file=f) diff --git a/scripts/bom-in-python/round_value_robin.py b/scripts/bom-in-python/round_value_robin.py index c7834420a7..536f5a1239 100644 --- a/scripts/bom-in-python/round_value_robin.py +++ b/scripts/bom-in-python/round_value_robin.py @@ -4,8 +4,10 @@ # Example: Round value robin, XML to XML conversion with partial value monging # +from __future__ import print_function + # Import the KiCad python helper module and the csv formatter -import ky_generic_netlist_reader +import kicad_netlist_reader import sys def checkvalue(self): @@ -51,21 +53,21 @@ def checkvalue(self): # Give components a new method for checking the values (this could easily be a # Company Part Number generator method instead) -ky_generic_netlist_reader.component.checkvalue = checkvalue +kicad_netlist_reader.comp.checkvalue = checkvalue # Generate an instance of a generic netlist, and load the netlist tree from # the command line option. If the file doesn't exist, execution will stop -net = ky_generic_netlist_reader.netlist(sys.argv[1]) +net = kicad_netlist_reader.netlist(sys.argv[1]) # Open a file to write to, if the file cannot be opened output to stdout # instead try: f = open(sys.argv[2], 'w') except IOError: - print >> sys.stderr, __file__, ":", e + print(__file__, ":", e, file=sys.stderr) f = stdout for c in net.components: c.checkvalue() -print >> f, net.formatXML() +print(net.formatXML(), file=f)