Warnings or errors for parts not in db. Build pref csv only once.

Emit errors (or warnings with --warn-only) when filling parts from the
preferred DB.

Libreoffice doesn't like being run in parallel, so build the
preferred-parts.csv file only once and then use it in each of the
parts list generator rules.

Signed-off-by: Keith Packard <keithp@keithp.com>

diff --git a/bin/fillpartscsv.py b/bin/fillpartscsv.py
index 675bb30428dd45db0fa70887a90bf8d3cb0f2b64..e502bae8177748891b4350ef5ae8d72856e3e5ef 100755 (executable)
--- a/bin/fillpartscsv.py
+++ b/bin/fillpartscsv.py
@@ -25,20 +25,29 @@ def main():
     parser = argparse.ArgumentParser();
     parser.add_argument("input", help="CSV format input file")
     parser.add_argument("-o", "--output", help="CSV format output file")
-    parser.add_argument("-p", "--preferred", help="Preferred parts database")
+    parser.add_argument("-p", "--preferred", required=True, help="Preferred parts database")
+    parser.add_argument("-w", "--warn-only", action="store_true",
+                        help="Make parts missing from the dB a warning rather than error")
     args = parser.parse_args()
 
-    if args.preferred:
-        preferred_name = args.preferred
-    else:
-        preferred_name = 'default'
-    preferred = parts.Parts(ods=preferred_name)
-    if args.input:
-        with open(args.input) as input:
-            my_parts = parts.Parts(csv_file=input)
-    else:
-        my_parts = parts.Parts(csv_file=sys.stdin)
-    my_parts.fill_values(preferred)
+    try:
+        preferred = parts.Parts(autofile=args.preferred)
+    except ValueError:
+        print('%s: Failed to load database' % args.preferred, file=sys.stderr)
+        sys.exit(1)
+    try:
+        if args.input:
+            parts_name = args.input
+            with open(args.input) as input:
+                my_parts = parts.Parts(csv_file=input)
+        else:
+            parts_name = "<stdin>"
+            my_parts = parts.Parts(csv_file=sys.stdin)
+    except ValueError:
+        print('%s: Failed to load parts list' % parts_name, file=sys.stderr)
+        sys.exit(1)
+    if not my_parts.fill_values(preferred, error=(not args.warn_only)):
+        sys.exit(1)
     if args.output:
         my_parts.export_csv(args.output)
     else:

diff --git a/bin/fillpartslist.py b/bin/fillpartslist.py
index afc5149db3f9a57264df87e63432412f2db288ce..119f663904974c958709c839b7bd7bc8f115cc2e 100755 (executable)
--- a/bin/fillpartslist.py
+++ b/bin/fillpartslist.py
@@ -25,20 +25,29 @@ def main():
     parser = argparse.ArgumentParser();
     parser.add_argument("input", help="Tab-delimited input file")
     parser.add_argument("-o", "--output", help="Tab-delimited output file")
-    parser.add_argument("-p", "--preferred", help="Preferred parts database")
+    parser.add_argument("-p", "--preferred", required=True, help="Preferred parts database")
+    parser.add_argument("-w", "--warn-only", action="store_true",
+                        help="Make parts missing from the dB a warning rather than error")
     args = parser.parse_args()
 
-    if args.preferred:
-        preferred_name = args.preferred
-    else:
-        preferred_name = 'default'
-    preferred = parts.Parts(ods=preferred_name)
-    if args.input:
-        with open(args.input) as input:
-            my_parts = parts.Parts(tab_file=input)
-    else:
-        my_parts = parts.Parts(tab_file=sys.stdin)
-    my_parts.fill_values(preferred)
+    try:
+        preferred = parts.Parts(autofile=args.preferred)
+    except ValueError:
+        print('%s: Failed to load database' % args.preferred, file=sys.stderr)
+        sys.exit(1)
+    try:
+        if args.input:
+            parts_name = args.input
+            with open(args.input) as input:
+                my_parts = parts.Parts(tab_file=input)
+        else:
+            parts_name = "<stdin>"
+            my_parts = parts.Parts(tab_file=sys.stdin)
+    except ValueError:
+        print('%s: Failed to load parts list' % parts_name, file=sys.stderr)
+        sys.exit(1)
+    if not my_parts.fill_values(preferred, error=(not args.warn_only)):
+        sys.exit(1)
     if args.output:
         my_parts.export_tab(args.output)
     else:

diff --git a/bin/parts.py b/bin/parts.py
index 3da77e3afae9289acd0a80b5036f5030ad17f69e..31d732af1fa373a29cbba1f6b85a85890949342c 100644 (file)
--- a/bin/parts.py
+++ b/bin/parts.py
@@ -113,12 +113,21 @@ class Parts():
 
     csv.register_dialect('excel-nl', 'excel', lineterminator='\n')
 
-    def __init__(self, parts=None, ods=None, csv=None, csv_file=None, tab=None, tab_file=None):
+    def __init__(self, parts=None, ods=None, csv=None, csv_file=None, tab=None, tab_file=None, autofile=None):
         if parts is not None:
             self.parts = parts
         else:
             self.parts = {}
 
+        if autofile is not None:
+            suffix = Path(autofile).suffix
+            if suffix == ".csv":
+                csv = autofile
+            elif suffix == ".ods":
+                ods = autofile
+            else:
+                tab = autofile
+
         if csv is not None:
             self.import_csv(csv)
 
@@ -126,8 +135,6 @@ class Parts():
             self.import_csv_file(csv_file)
 
         if ods is not None:
-            if ods == 'default':
-                ods = Path(sys.path[0]) / '..' / 'preferred-parts.ods'
             self.import_ods(ods)
 
         if tab is not None:
@@ -166,15 +173,16 @@ class Parts():
     # Import from an ODS (Libreoffice calc) file
     def import_ods(self, inname):
         with tempfile.TemporaryDirectory() as dir:
+            print('temporary dir %s' % dir)
             outname = Path(dir) / Path(inname).with_suffix('.csv').name
 
-            ret = subprocess.run(('libreoffice', '--headless', '--convert-to', 'csv', inname, '--outdir', dir), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
-            try:
-                if ret.returncode != 0:
-                    raise ValueEror('cannot convert "%s": ' % (inname, ret.stderr))
-                return self.import_csv(outname)
-            finally:
-                os.remove(outname)
+            ret = subprocess.run(('libreoffice', '--headless', '--nolockcheck', '--convert-to', 'csv', inname, '--outdir', dir),
+#                                 stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+                                 stderr=None, stdout=None)
+            if ret.returncode != 0:
+                print('convert stderr "%s" stdout "%s"' % (ret.stderr, ret.stdout))
+                raise ValueError('cannot convert "%s": %s' % (inname, ret.stderr + ret.stdout))
+            return self.import_csv(outname)
 
     # Import from a tab-deliminted file object
     def import_tab_file(self, infile):
@@ -192,12 +200,22 @@ class Parts():
             self.import_tab_file(infile)
 
     # Fill missing values from the preferred parts list
-    def fill_values(self, preferred_parts):
+    def fill_values(self, preferred_parts, error=False):
+        ret = True
+        if error:
+            message = 'Error'
+        else:
+            message = 'Warning'
         for key in self.parts:
             part = self.parts[key]
             pref = preferred_parts.get(key)
-            if pref is not None:
+            if pref is None:
+                print("%s: design part %r not present in database" % (message, key), file=sys.stderr)
+                if error:
+                    ret = False
+            else:
                 part.fill_values(pref)
+        return ret
 
     # Compute the set of all attributes in all of the parts
     def attrs_set(self):

diff --git a/pcb-rnd.mk b/pcb-rnd.mk
index 3c7e7ca13f94f91bb2cdf065dea4be6a3cf5da29..2de9604c5784b8a916c7c6bd7ae2fd7c7972d7f3 100644 (file)
--- a/pcb-rnd.mk
+++ b/pcb-rnd.mk
@@ -15,7 +15,7 @@ endif
 
 CONFIG=gafrc attribs project.lht
 
-all:   drc partslist partslist.csv pcb
+all:   drc partslist
 
 drc:   $(PROJECT).drc
 
@@ -24,15 +24,11 @@ $(PROJECT).drc: $(SCHEMATICS) Makefile $(CONFIG)
 
 partslists: partslist partslist.csv partslist-dk.csv partslist-check.dk partslist-mouser.csv partslist-other.csv
 
-partslist:     $(SCHEMATICS) Makefile $(AM)/preferred-parts.ods $(CONFIG)
-       lepton-netlist -g bom -o $(PROJECT).unsorted $(SCHEMATICS)
-       $(AM)/bin/fillpartslist.py $(PROJECT).unsorted --output $@ --preferred $(AM)/preferred-parts.ods
-       rm -f $(PROJECT).unsorted
+partslist:     $(PROJECT)-parts.tab preferred-parts.csv $(CONFIG)
+       $(AM)/bin/fillpartslist.py --warn-only $(PROJECT)-parts.tab --output $@ --preferred preferred-parts.csv
 
-partslist.csv: $(SCHEMATICS) Makefile $(AM)/preferred-parts.ods $(CONFIG)
-       lepton-netlist -L $(SCHEME) -g bomAM -o $(PROJECT).csvtmp $(SCHEMATICS)
-       $(AM)/bin/fillpartscsv.py $(PROJECT).csvtmp --output $@ --preferred $(AM)/preferred-parts.ods
-       rm -f $(PROJECT).csvtmp
+partslist.csv: $(PROJECT)-parts.csv preferred-parts.csv $(CONFIG)
+       $(AM)/bin/fillpartscsv.py $(PROJECT)-parts.csv --output $@ --preferred preferred-parts.csv
 
 partslist-dk.csv: partslist.csv
        $(AM)/bin/partslist-vendor --vendor digikey partslist.csv > $@
@@ -52,6 +48,15 @@ $(PROJECT)-seeed.csv: partslist.csv
 $(PROJECT)-goldphoenix.csv: partslist.csv
        $(AM)/bin/partslist-vendor --vendor goldphoenix partslist.csv > $@
 
+$(PROJECT)-parts.tab: $(SCHEMATICS) Makefile
+       lepton-netlist -g bom -o $@ $(SCHEMATICS)
+
+$(PROJECT)-parts.csv: $(SCHEMATICS) Makefile
+       lepton-netlist -L $(SCHEME) -g bomAM -o $@ $(SCHEMATICS)
+
+preferred-parts.csv: $(AM)/preferred-parts.ods
+       libreoffice --headless --convert-to csv $^ --outdir .
+
 pcb:   $(SCHEMATICS) Makefile $(CONFIG)
        lepton-netlist -g tEDAx -o $(PROJECT).tdx $(SCHEMATICS)
        echo 'LoadTedaxFrom(netlist, $(PROJECT).tdx); SaveTo(Layout)' | \
@@ -120,6 +125,7 @@ clean:
        rm -f $(PROJECT)-seeed.zip $(PROJECT)-seeed.csv
        rm -f $(PROJECT)-goldphoenix.zip $(PROJECT)-goldphoenix.csv
        rm -f $(PROJECT)*.ps $(PROJECT)*.pdf $(PROJECT)-bom.csv
+       rm -f $(PROJECT)-parts.csv $(PROJECT)-parts.tab preferred-parts.csv
        rm -f *.scad
        rm -fr out
 

diff --git a/pcb.mk b/pcb.mk
index b5b95029d1081e441d6f71e00e3fbb983de657e1..b6fe2eac8feca23c05a862c646b368f181f7aca8 100644 (file)
--- a/pcb.mk
+++ b/pcb.mk
@@ -15,7 +15,7 @@ endif
 
 CONFIG=gafrc attribs project
 
-all:   drc partslist partslist.csv pcb
+all:   drc partslist
 
 drc:   $(PROJECT).drc
 
@@ -24,15 +24,11 @@ $(PROJECT).drc: $(SCHEMATICS) Makefile $(CONFIG)
 
 partslists: partslist partslist.csv partslist-dk.csv partslist-check.dk partslist-mouser.csv partslist-other.csv
 
-partslist:     $(SCHEMATICS) Makefile $(AM)/preferred-parts.ods $(CONFIG)
-       gnetlist -g bom -o $(PROJECT).unsorted $(SCHEMATICS)
-       $(AM)/bin/fillpartslist.py $(PROJECT).unsorted --output $@ --preferred $(AM)/preferred-parts.ods
-       rm -f $(PROJECT).unsorted
+partslist:     $(PROJECT)-parts.tab preferred-parts.csv
+       $(AM)/bin/fillpartslist.py $(PROJECT)-parts.tab --output $@ --preferred preferred-parts.csv
 
-partslist.csv: $(SCHEMATICS) Makefile $(AM)/preferred-parts.ods $(CONFIG)
-       gnetlist -L $(SCHEME) -g bomAM -o $(PROJECT).csvtmp $(SCHEMATICS)
-       $(AM)/bin/fillpartscsv.py $(PROJECT).csvtmp --output $@ --preferred $(AM)/preferred-parts.ods
-       rm -f $(PROJECT).csvtmp
+partslist.csv: $(PROJECT)-parts.csv preferred-parts.csv
+       $(AM)/bin/fillpartscsv.py $(PROJECT)-parts.csv --output $@ --preferred preferred-parts.csv
 
 partslist-dk.csv: partslist.csv
        $(AM)/bin/partslist-vendor --vendor digikey partslist.csv > $@
@@ -55,6 +51,15 @@ $(PROJECT)-goldphoenix.csv: partslist.csv
 pcb:   $(SCHEMATICS) Makefile $(CONFIG)
        lepton-sch2pcb project
 
+$(PROJECT)-parts.tab: $(SCHEMATICS) Makefile $(CONFIG)
+       lepton-netlist -g bom -o $@ $(SCHEMATICS)
+
+$(PROJECT)-parts.csv: $(SCHEMATICS) Makefile $(CONFIG)
+       lepton-netlist -L $(SCHEME) -g bomAM -o $@ $(SCHEMATICS)
+
+preferred-parts.csv: $(AM)/preferred-parts.ods
+       libreoffice --headless --convert-to csv $^ --outdir .
+
 $(PROJECT).xy: $(PROJECT).pcb $(CONFIG)
        pcb -x bom $(PROJECT).pcb
 
@@ -246,6 +251,7 @@ clean:
        rm -f $(PROJECT)-seeed.zip $(PROJECT)-seeed.csv
        rm -f $(PROJECT)-goldphoenix.zip $(PROJECT)-goldphoenix.csv
        rm -f $(PROJECT)*.ps $(PROJECT)*.pdf
+       rm -f $(PROJECT)-parts.tab $(PROJECT)-parts.csv preferred-parts.csv
 
 muffins: muffin-6570.pdf muffin-5267.pdf muffin-keithp.pdf