From 05b848e5ec8a802eefc8d0f252eb8dad66246fb9 Mon Sep 17 00:00:00 2001 From: Keith Packard Date: Mon, 10 Feb 2025 14:43:14 -0800 Subject: [PATCH] bin: Add python programs to handle CSV (and ODS) formatted parts files Tools necessary to transition from tab-separated file to a spreadsheet Signed-off-by: Keith Packard --- bin/fillpartscsv.py | 30 +++++ bin/fillpartslist.py | 29 +++++ bin/parts.py | 291 +++++++++++++++++++++++++++++++++++++++++++ bin/tabtocsv.py | 18 +++ 4 files changed, 368 insertions(+) create mode 100755 bin/fillpartscsv.py create mode 100755 bin/fillpartslist.py create mode 100644 bin/parts.py create mode 100755 bin/tabtocsv.py diff --git a/bin/fillpartscsv.py b/bin/fillpartscsv.py new file mode 100755 index 0000000..4f2d24f --- /dev/null +++ b/bin/fillpartscsv.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +# +# Copyright © 2025 Keith Packard +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. +# + +import parts; +import csv; +import sys; + +def main(): + preferred = parts.Parts(ods='default') + my_parts = parts.Parts(csv_file=sys.stdin) + my_parts.fill_values(preferred) + my_parts.export_csv_file(sys.stdout) + +main() diff --git a/bin/fillpartslist.py b/bin/fillpartslist.py new file mode 100755 index 0000000..00324db --- /dev/null +++ b/bin/fillpartslist.py @@ -0,0 +1,29 @@ +#!/usr/bin/python3 +# +# Copyright © 2025 Keith Packard +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. +# + +import parts; +import sys; + +def main(): + preferred = parts.Parts(ods='default') + my_parts = parts.Parts(tab_file=sys.stdin) + my_parts.fill_values(preferred) + my_parts.export_tab_file(sys.stdout) + +main() diff --git a/bin/parts.py b/bin/parts.py new file mode 100644 index 0000000..aca296e --- /dev/null +++ b/bin/parts.py @@ -0,0 +1,291 @@ +# +# Copyright © 2025 Keith Packard +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. +# + +""" +AltusMetrum collection of classes +""" + +import csv; +import tempfile; +import subprocess; +import os +import sys +import functools +from pathlib import Path; +import re; + +# These attributes form the 'key' used to uniquely identify the part +key_attrs = ('device', 'value', 'footprint') + +# This is the preferred order when writing a CSV file +pref_order = ('device', 'value', 'footprint', 'loadstatus', 'provided', 'mfg', + 'mfg_part_number', 'vendor', 'vendor_part_number', 'quantity', 'refdes') + +value_pattern=r'([0-9]+)(\.[0-9]*)?([kmMupng]?)(F|H|Hz)?' + +def numeric_value(a): + m = re.fullmatch(value_pattern, a, flags=re.IGNORECASE) + if m: + n = m.group(1) + if m.group(2): + n += m.group(2) + number = float(n) + scale = m.group(3) + if scale == 'G': + number *= 1000000000 + elif scale == 'M': + number *= 1000000 + elif scale == 'k': + number *= 1000 + elif scale == 'm': + number /= 1000 + elif scale == 'u' or scale == 'µ': + number /= 1000000 + elif scale == 'p': + number /= 1000000000 + return number + return None + +def value_cmp(a, b): + na = numeric_value(a) + nb = numeric_value(b) + if na and nb: + if na < nb: + return -1 + if na > nb: + return 1 + return 0 + if na: + return -1 + if nb: + return 1 + if a < b: + return -1 + if b > a: + return 1 + return 0 + +def str_cmp(a,b): + if a < b: + return -1 + if a > b: + return 1 + return 0 + +def key_cmp(a, b): + c = str_cmp(a[0], b[0]) + if c: + return c + c = value_cmp(a[1], b[1]) + if c: + return c + return str_cmp(a[2], b[2]) + +class Part(): + """ + A single part containing a dictionary with all of the attributes + """ + + # Initialize the object, optionally incorporating a value from a list + def __init__(self, attrs=None, values=None, keys=None): + if attrs is not None: + self.attrs = attrs + else: + self.attrs = {} + + if values is not None: + for i in range(len(values)): + if values[i]: + self.attrs[keys[i]] = values[i] + + # Get an attribute value, returning None for + # missing attributes + def get(self, attr): + if attr in self.attrs: + return self.attrs[attr] + return None + + # Get an attribute value, returning 'unknown' for + # missing attributes + def get_unknown(self, attr): + v = self.get(attr) + if v is None: + v = 'unknown' + return v + + # Check for a missing or unknown attribute + def missing(self, attr): + v = self.get(attr) + return v is None or v == 'unknown' + + # Set an attribute value + def set(self, attr, value): + if value is None: + if attr in self.attrs: + del self.attrs[attr] + else: + self.attrs[attr] = value + + # Compute the key tuple (device, value, footprint) + def key(self): + return tuple(map(self.get, key_attrs)) + + # Return a set of all attributes in the part + def attrs_set(self): + return set(self.attrs) + + # Fill in missing values from the preferred part + def fill_values(self, pref_part): + for key in pref_part.attrs: + if self.missing(key): + self.set(key, pref_part.get(key)) + + def __repr__(self): + return str(self.attrs) + +class Parts(): + + """ + A parts list, indexed by the part key (dice, value, footprint) + """ + + csv_dialect = csv.register_dialect('excel-nl', 'excel', lineterminator='\n') + + def __init__(self, parts=None, ods=None, csv=None, csv_file=None, tab=None, tab_file=None): + if parts is not None: + self.parts = parts + else: + self.parts = {} + + if csv is not None: + self.import_csv(csv) + + if csv_file is not None: + self.import_csv_file(csv_file) + + if ods is not None: + if ods == 'default': + ods = Path(sys.path[0]) / '..' / 'preferred-parts.ods' + self.import_ods(ods) + + if tab is not None: + self.import_tab(tab) + + if tab_file is not None: + self.import_tab_file(tab_file) + + # Lookup a part by key + def get(self, key): + key = tuple(key) + if key in self.parts: + return self.parts[key] + return None + + # Add/replace a part, computing the key + def set(self, part): + key = part.key() + self.parts[key] = part + + # Import from a CSV file object + def import_csv_file(self, infile): + csvreader = csv.reader(infile) + csvkeys = None + for csvline in csvreader: + if csvkeys is None: + csvkeys = csvline + else: + self.set(Part(values=csvline, keys=csvkeys)) + + # Import from a CSV file + def import_csv(self, inname): + with open(inname) as infile: + self.import_csv_file(infile) + + # Import from an ODS (Libreoffice calc) file + def import_ods(self, inname): + with tempfile.TemporaryDirectory() as dir: + outname = Path(dir) / Path(inname).with_suffix('.csv').name + + ret = subprocess.run(('libreoffice', '--headless', '--convert-to', 'csv', inname, '--outdir', dir), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) + try: + if ret.returncode != 0: + raise ValueEror('cannot convert "%s": ' % (inname, ret.stderr)) + return self.import_csv(outname) + finally: + os.remove(outname) + + # Import from a tab-deliminted file object + def import_tab_file(self, infile): + keys=None + for line in infile: + values = tuple(map(str.strip, line.strip().split('\t'))) + if keys is None: + keys = values + else: + self.set(Part(values=values, keys=keys)) + + # Import from a tab-delimited file + def import_tab(self, inname): + with open(inname) as infile: + self.import_tab_file(infile) + + # Fill missing values from the preferred parts list + def fill_values(self, preferred_parts): + for key in self.parts: + part = self.parts[key] + pref = preferred_parts.get(key) + if pref is not None: + part.fill_values(pref) + + # Compute the set of all attributes in all of the parts + def attrs_set(self): + ret = set(key_attrs) + for key in self.parts: + ret = ret | self.get(key).attrs_set() + return ret + + # Generate a tuple of all attributes in the preferred CSV order + def attrs_tuple(self): + s = self.attrs_set() + t = key_attrs + for p in pref_order: + if p in s and p not in t: + t = t + (p,) + for i in s: + if i not in t: + t = t + (i,) + return t + + # Export to a CSV file + def export_csv_file(self, outfile): + csvwriter = csv.writer(outfile, dialect='excel-nl') + attrs = self.attrs_tuple() + csvwriter.writerow(attrs) + keys = sorted(list(self.parts), key=functools.cmp_to_key(key_cmp)) + for key in keys: + part = self.get(key) + csvwriter.writerow(tuple(map(part.get_unknown, attrs))) + + # Export to a tab-delimited file + def export_tab_file(self, outfile): + attrs = self.attrs_tuple() + print("\t".join(attrs), file=outfile) + keys = sorted(list(self.parts), key=functools.cmp_to_key(key_cmp)) + for key in keys: + part = self.get(key) + print("\t".join(tuple(map(part.get_unknown, attrs))), file=outfile) diff --git a/bin/tabtocsv.py b/bin/tabtocsv.py new file mode 100755 index 0000000..aa1baad --- /dev/null +++ b/bin/tabtocsv.py @@ -0,0 +1,18 @@ +#!/usr/bin/python3 + +import csv; +import sys; + +def tabtocsv(infile, outfile): + csvwriter = csv.writer(outfile) + for line in infile: + fields=line.strip().split('\t') + csvwriter.writerow(fields) + + +def main(): + infile = sys.stdin + outfile = sys.stdout + tabtocsv(infile, outfile) + +main() -- 2.47.2