From 1889122829dc2a723f1e3d465f62685b8fbd52cd Mon Sep 17 00:00:00 2001 From: Keith Packard Date: Tue, 11 Feb 2025 20:45:01 -0800 Subject: [PATCH] Warnings or errors for parts not in db. Build pref csv only once. Emit errors (or warnings with --warn-only) when filling parts from the preferred DB. Libreoffice doesn't like being run in parallel, so build the preferred-parts.csv file only once and then use it in each of the parts list generator rules. Signed-off-by: Keith Packard --- bin/fillpartscsv.py | 33 +++++++++++++++++++++------------ bin/fillpartslist.py | 33 +++++++++++++++++++++------------ bin/parts.py | 42 ++++++++++++++++++++++++++++++------------ pcb-rnd.mk | 24 +++++++++++++++--------- pcb.mk | 24 +++++++++++++++--------- 5 files changed, 102 insertions(+), 54 deletions(-) diff --git a/bin/fillpartscsv.py b/bin/fillpartscsv.py index 675bb30..e502bae 100755 --- a/bin/fillpartscsv.py +++ b/bin/fillpartscsv.py @@ -25,20 +25,29 @@ def main(): parser = argparse.ArgumentParser(); parser.add_argument("input", help="CSV format input file") parser.add_argument("-o", "--output", help="CSV format output file") - parser.add_argument("-p", "--preferred", help="Preferred parts database") + parser.add_argument("-p", "--preferred", required=True, help="Preferred parts database") + parser.add_argument("-w", "--warn-only", action="store_true", + help="Make parts missing from the dB a warning rather than error") args = parser.parse_args() - if args.preferred: - preferred_name = args.preferred - else: - preferred_name = 'default' - preferred = parts.Parts(ods=preferred_name) - if args.input: - with open(args.input) as input: - my_parts = parts.Parts(csv_file=input) - else: - my_parts = parts.Parts(csv_file=sys.stdin) - my_parts.fill_values(preferred) + try: + preferred = parts.Parts(autofile=args.preferred) + except ValueError: + print('%s: Failed to load database' % args.preferred, file=sys.stderr) + sys.exit(1) + try: + if args.input: + parts_name = args.input + with open(args.input) as input: + my_parts = parts.Parts(csv_file=input) + else: + parts_name = "" + my_parts = parts.Parts(csv_file=sys.stdin) + except ValueError: + print('%s: Failed to load parts list' % parts_name, file=sys.stderr) + sys.exit(1) + if not my_parts.fill_values(preferred, error=(not args.warn_only)): + sys.exit(1) if args.output: my_parts.export_csv(args.output) else: diff --git a/bin/fillpartslist.py b/bin/fillpartslist.py index afc5149..119f663 100755 --- a/bin/fillpartslist.py +++ b/bin/fillpartslist.py @@ -25,20 +25,29 @@ def main(): parser = argparse.ArgumentParser(); parser.add_argument("input", help="Tab-delimited input file") parser.add_argument("-o", "--output", help="Tab-delimited output file") - parser.add_argument("-p", "--preferred", help="Preferred parts database") + parser.add_argument("-p", "--preferred", required=True, help="Preferred parts database") + parser.add_argument("-w", "--warn-only", action="store_true", + help="Make parts missing from the dB a warning rather than error") args = parser.parse_args() - if args.preferred: - preferred_name = args.preferred - else: - preferred_name = 'default' - preferred = parts.Parts(ods=preferred_name) - if args.input: - with open(args.input) as input: - my_parts = parts.Parts(tab_file=input) - else: - my_parts = parts.Parts(tab_file=sys.stdin) - my_parts.fill_values(preferred) + try: + preferred = parts.Parts(autofile=args.preferred) + except ValueError: + print('%s: Failed to load database' % args.preferred, file=sys.stderr) + sys.exit(1) + try: + if args.input: + parts_name = args.input + with open(args.input) as input: + my_parts = parts.Parts(tab_file=input) + else: + parts_name = "" + my_parts = parts.Parts(tab_file=sys.stdin) + except ValueError: + print('%s: Failed to load parts list' % parts_name, file=sys.stderr) + sys.exit(1) + if not my_parts.fill_values(preferred, error=(not args.warn_only)): + sys.exit(1) if args.output: my_parts.export_tab(args.output) else: diff --git a/bin/parts.py b/bin/parts.py index 3da77e3..31d732a 100644 --- a/bin/parts.py +++ b/bin/parts.py @@ -113,12 +113,21 @@ class Parts(): csv.register_dialect('excel-nl', 'excel', lineterminator='\n') - def __init__(self, parts=None, ods=None, csv=None, csv_file=None, tab=None, tab_file=None): + def __init__(self, parts=None, ods=None, csv=None, csv_file=None, tab=None, tab_file=None, autofile=None): if parts is not None: self.parts = parts else: self.parts = {} + if autofile is not None: + suffix = Path(autofile).suffix + if suffix == ".csv": + csv = autofile + elif suffix == ".ods": + ods = autofile + else: + tab = autofile + if csv is not None: self.import_csv(csv) @@ -126,8 +135,6 @@ class Parts(): self.import_csv_file(csv_file) if ods is not None: - if ods == 'default': - ods = Path(sys.path[0]) / '..' / 'preferred-parts.ods' self.import_ods(ods) if tab is not None: @@ -166,15 +173,16 @@ class Parts(): # Import from an ODS (Libreoffice calc) file def import_ods(self, inname): with tempfile.TemporaryDirectory() as dir: + print('temporary dir %s' % dir) outname = Path(dir) / Path(inname).with_suffix('.csv').name - ret = subprocess.run(('libreoffice', '--headless', '--convert-to', 'csv', inname, '--outdir', dir), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) - try: - if ret.returncode != 0: - raise ValueEror('cannot convert "%s": ' % (inname, ret.stderr)) - return self.import_csv(outname) - finally: - os.remove(outname) + ret = subprocess.run(('libreoffice', '--headless', '--nolockcheck', '--convert-to', 'csv', inname, '--outdir', dir), +# stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stderr=None, stdout=None) + if ret.returncode != 0: + print('convert stderr "%s" stdout "%s"' % (ret.stderr, ret.stdout)) + raise ValueError('cannot convert "%s": %s' % (inname, ret.stderr + ret.stdout)) + return self.import_csv(outname) # Import from a tab-deliminted file object def import_tab_file(self, infile): @@ -192,12 +200,22 @@ class Parts(): self.import_tab_file(infile) # Fill missing values from the preferred parts list - def fill_values(self, preferred_parts): + def fill_values(self, preferred_parts, error=False): + ret = True + if error: + message = 'Error' + else: + message = 'Warning' for key in self.parts: part = self.parts[key] pref = preferred_parts.get(key) - if pref is not None: + if pref is None: + print("%s: design part %r not present in database" % (message, key), file=sys.stderr) + if error: + ret = False + else: part.fill_values(pref) + return ret # Compute the set of all attributes in all of the parts def attrs_set(self): diff --git a/pcb-rnd.mk b/pcb-rnd.mk index 3c7e7ca..2de9604 100644 --- a/pcb-rnd.mk +++ b/pcb-rnd.mk @@ -15,7 +15,7 @@ endif CONFIG=gafrc attribs project.lht -all: drc partslist partslist.csv pcb +all: drc partslist drc: $(PROJECT).drc @@ -24,15 +24,11 @@ $(PROJECT).drc: $(SCHEMATICS) Makefile $(CONFIG) partslists: partslist partslist.csv partslist-dk.csv partslist-check.dk partslist-mouser.csv partslist-other.csv -partslist: $(SCHEMATICS) Makefile $(AM)/preferred-parts.ods $(CONFIG) - lepton-netlist -g bom -o $(PROJECT).unsorted $(SCHEMATICS) - $(AM)/bin/fillpartslist.py $(PROJECT).unsorted --output $@ --preferred $(AM)/preferred-parts.ods - rm -f $(PROJECT).unsorted +partslist: $(PROJECT)-parts.tab preferred-parts.csv $(CONFIG) + $(AM)/bin/fillpartslist.py --warn-only $(PROJECT)-parts.tab --output $@ --preferred preferred-parts.csv -partslist.csv: $(SCHEMATICS) Makefile $(AM)/preferred-parts.ods $(CONFIG) - lepton-netlist -L $(SCHEME) -g bomAM -o $(PROJECT).csvtmp $(SCHEMATICS) - $(AM)/bin/fillpartscsv.py $(PROJECT).csvtmp --output $@ --preferred $(AM)/preferred-parts.ods - rm -f $(PROJECT).csvtmp +partslist.csv: $(PROJECT)-parts.csv preferred-parts.csv $(CONFIG) + $(AM)/bin/fillpartscsv.py $(PROJECT)-parts.csv --output $@ --preferred preferred-parts.csv partslist-dk.csv: partslist.csv $(AM)/bin/partslist-vendor --vendor digikey partslist.csv > $@ @@ -52,6 +48,15 @@ $(PROJECT)-seeed.csv: partslist.csv $(PROJECT)-goldphoenix.csv: partslist.csv $(AM)/bin/partslist-vendor --vendor goldphoenix partslist.csv > $@ +$(PROJECT)-parts.tab: $(SCHEMATICS) Makefile + lepton-netlist -g bom -o $@ $(SCHEMATICS) + +$(PROJECT)-parts.csv: $(SCHEMATICS) Makefile + lepton-netlist -L $(SCHEME) -g bomAM -o $@ $(SCHEMATICS) + +preferred-parts.csv: $(AM)/preferred-parts.ods + libreoffice --headless --convert-to csv $^ --outdir . + pcb: $(SCHEMATICS) Makefile $(CONFIG) lepton-netlist -g tEDAx -o $(PROJECT).tdx $(SCHEMATICS) echo 'LoadTedaxFrom(netlist, $(PROJECT).tdx); SaveTo(Layout)' | \ @@ -120,6 +125,7 @@ clean: rm -f $(PROJECT)-seeed.zip $(PROJECT)-seeed.csv rm -f $(PROJECT)-goldphoenix.zip $(PROJECT)-goldphoenix.csv rm -f $(PROJECT)*.ps $(PROJECT)*.pdf $(PROJECT)-bom.csv + rm -f $(PROJECT)-parts.csv $(PROJECT)-parts.tab preferred-parts.csv rm -f *.scad rm -fr out diff --git a/pcb.mk b/pcb.mk index b5b9502..b6fe2ea 100644 --- a/pcb.mk +++ b/pcb.mk @@ -15,7 +15,7 @@ endif CONFIG=gafrc attribs project -all: drc partslist partslist.csv pcb +all: drc partslist drc: $(PROJECT).drc @@ -24,15 +24,11 @@ $(PROJECT).drc: $(SCHEMATICS) Makefile $(CONFIG) partslists: partslist partslist.csv partslist-dk.csv partslist-check.dk partslist-mouser.csv partslist-other.csv -partslist: $(SCHEMATICS) Makefile $(AM)/preferred-parts.ods $(CONFIG) - gnetlist -g bom -o $(PROJECT).unsorted $(SCHEMATICS) - $(AM)/bin/fillpartslist.py $(PROJECT).unsorted --output $@ --preferred $(AM)/preferred-parts.ods - rm -f $(PROJECT).unsorted +partslist: $(PROJECT)-parts.tab preferred-parts.csv + $(AM)/bin/fillpartslist.py $(PROJECT)-parts.tab --output $@ --preferred preferred-parts.csv -partslist.csv: $(SCHEMATICS) Makefile $(AM)/preferred-parts.ods $(CONFIG) - gnetlist -L $(SCHEME) -g bomAM -o $(PROJECT).csvtmp $(SCHEMATICS) - $(AM)/bin/fillpartscsv.py $(PROJECT).csvtmp --output $@ --preferred $(AM)/preferred-parts.ods - rm -f $(PROJECT).csvtmp +partslist.csv: $(PROJECT)-parts.csv preferred-parts.csv + $(AM)/bin/fillpartscsv.py $(PROJECT)-parts.csv --output $@ --preferred preferred-parts.csv partslist-dk.csv: partslist.csv $(AM)/bin/partslist-vendor --vendor digikey partslist.csv > $@ @@ -55,6 +51,15 @@ $(PROJECT)-goldphoenix.csv: partslist.csv pcb: $(SCHEMATICS) Makefile $(CONFIG) lepton-sch2pcb project +$(PROJECT)-parts.tab: $(SCHEMATICS) Makefile $(CONFIG) + lepton-netlist -g bom -o $@ $(SCHEMATICS) + +$(PROJECT)-parts.csv: $(SCHEMATICS) Makefile $(CONFIG) + lepton-netlist -L $(SCHEME) -g bomAM -o $@ $(SCHEMATICS) + +preferred-parts.csv: $(AM)/preferred-parts.ods + libreoffice --headless --convert-to csv $^ --outdir . + $(PROJECT).xy: $(PROJECT).pcb $(CONFIG) pcb -x bom $(PROJECT).pcb @@ -246,6 +251,7 @@ clean: rm -f $(PROJECT)-seeed.zip $(PROJECT)-seeed.csv rm -f $(PROJECT)-goldphoenix.zip $(PROJECT)-goldphoenix.csv rm -f $(PROJECT)*.ps $(PROJECT)*.pdf + rm -f $(PROJECT)-parts.tab $(PROJECT)-parts.csv preferred-parts.csv muffins: muffin-6570.pdf muffin-5267.pdf muffin-keithp.pdf -- 2.47.2