Improve scalability by splitting the distributor cache. Additional cleanup and

generalizations.

- boom/Makefile, dist/dk/Makefile: the "files with everything" are now
  called dist/db/all.* instead of dist/db/digi-key.*
- boom/manu/panasonic/Makefile, boom/manu/stackpole/Makefile,
  boom/manu/yageo/Makefile: changed digi-key.equ to all.equ
- boom/Makefile: there can now be a list of distributors
- boom/Makefile (tar): new target to generate a snapshot of the database
- boom/Makefile (PHONY): added "spotless"
- boom/dist/dk/Makefile: do not merge catalog listings so that we don't have
  to search a huge cache with O(n^2)
- boom/dist/dk/Makefile (clean, rebuild, update, spotless): targets for
  different degrees of rebuilding
- boom/dist/dk/dk-db.pl: added progress report output

boom/Makefile

@@ -2,13 +2,20 @@ BOOM=PATH=/home/moko/svn.openmoko.org/trunk/eda/boom:../boom:$$PATH boom
 
 KITS = 1
 
-INV = dist/dk/db/digi-key.inv
-EQU = dist/dk/db/digi-key.equ
-DSC = dist/dk/db/digi-key.dsc
+DIST = dk
+EQU = $(DIST:%=dist/%/db/all.equ)
+INV = $(DIST:%=dist/%/db/all.inv)
+DSC = $(DIST:%=dist/%/db/all.dsc)
 CHR = manu/panasonic/panasonic.chr manu/stackpole/stackpole.chr \
       manu/yageo/yageo.chr
 
-.PHONY:		all show again
+DB = $(CHR) \
+    $(DIST:%=dist/%/db/*.equ) \
+    $(DIST:%=dist/%/db/*.inv) \
+    $(DIST:%=dist/%/db/*.dsc)
+
+
+.PHONY:		all show again spotless tar
 
 all:		show
 
@@ -30,3 +37,7 @@ test.lst:	test.mbq
 
 spotless:
 		rm -f test.lst test.par test.ord
+
+tar:
+		tar cfj boom-db.tar.bz2 $(DB)
+		

boom/dist/dk/Makefile

@@ -1,27 +1,75 @@
-CACHE = query.data
-EQUS = panasonic-erj stackpole-rmcf yageo-rc
+MASTER = db/all.equ db/all.dsc db/all.inv
+PARTS = panasonic-erj stackpole-rmcf yageo-rc
 
-.PHONY:			all
+.SECONDARY:		# prevent *.cache from being automatically deleted
 
-all:			db/digi-key.dsc db/digi-key.inv
+.PHONY:			all clean rebuild update spotless
 
-$(CACHE):		db/digi-key.equ
+all:			$(MASTER) \
+			  $(PARTS:%=db/%.equ) \
+			  $(PARTS:%=db/%.inv) \
+			  $(PARTS:%=db/%.dsc) 
+
+db/%.cache:		db/%.equ
 			awk '/^#END/ { exit } /^DIGI-KEY / { print $$2 }' $< | \
 			  perl ./dk-db.pl query \
-			  `[ -r $(CACHE) ] && echo '' -i $(CACHE)` >_$@ || \
-			  { rm -f $@ _$@; exit 1; }
-			mv _$@ $@
+			  `[ -r $@ ] && echo '' -i $@` >$(@)_ || \
+			  { rm -f $@ $(@)_; exit 1; }
+			mv $(@)_ $@
 
-db/digi-key.equ:	$(EQUS:%=db/%.equ)
-			cat $^ >$@ || { rm -rf $@; exit 1; }
+db:
+			mkdir -p db
 
-db/digi-key.dsc:	$(CACHE)
-			perl ./dk-db.pl dsc $< >$@ || \
-			  { rm -f $@; exit 1; }
+#
+# We need "db" to create the directory. However, this also means that any
+# change in the db/ directory triggers a rebuild of all.equ. This is cheap, so
+# this is nothing but a small cosmetic issue.
+#
 
-db/digi-key.inv:	$(CACHE)
-			perl ./dk-db.pl inv $< >$@ || \
-			  { rm -f $@; exit 1; }
+db/all.equ:		db Makefile
+			for n in $(PARTS); do echo "include $$n.equ"; done >$@
+
+db/all.inv:		Makefile
+			for n in $(PARTS); do echo "include $$n.inv"; done >$@
+
+db/all.dsc:		Makefile
+			for n in $(PARTS); do echo "include $$n.dsc"; done >$@
 
 db/%.equ:		%.catq
 			perl ./dk-catq.pl $< >$@ || { rm -rf $@; exit 1; }
+
+db/%.inv:		db/%.cache
+			perl ./dk-db.pl inv $< >$@ || { rm -f $@; exit 1; }
+
+db/%.dsc:		db/%.cache
+			perl ./dk-db.pl dsc $< >$@ || { rm -f $@; exit 1; }
+
+
+# --- Cleanup -----------------------------------------------------------------
+
+#
+# We have various levels of cleanup:
+#
+# - clean: remove left-over temporary files
+# - rebuild: remove everything that can be rebuilt from cached data
+# - update: remove the short-lived inventory and pricing cache too
+# - again: remove also the long-lived equivalences cache
+#
+# "clean" does not disturb use of the database. The run-time cost of recovering
+# from "rebuild" is small. What takes a lot of time is "update". The additional
+# cost of "spotless" is small again.
+#
+
+clean:
+			rm -f $(PARTS:%=db/%.cache_)
+
+rebuild:		clean
+			rm -f $(MASTER)
+			rm -f $(PARTS:%=db/%.inv)
+			rm -f $(PARTS:%=db/%.dsc)
+
+update:			rebuild
+			rm -f $(PARTS:%=db/%.cache)
+
+spotless:		update
+			rm -f $(PARTS:%=db/%.equ)

boom/dist/dk/dk-db.pl

@@ -63,6 +63,10 @@ if ($mode eq "query") {
 	next if /^\s*$/;
 	next if /^\s/;
 	s/\s.*//;
+
+	$n++;
+	print STDERR "$n\r" unless $n % 100;
+
 	next if $old =~ m#align=right>Digi-Key Part Number</th><td>$_</td#;
 	push(@pn, $_);
 	&flush if @pn > 1000;

boom/manu/panasonic/Makefile

@@ -1,4 +1,4 @@
 BOOM=PATH=/home/moko/svn.openmoko.org/trunk/eda/boom:../boom:$$PATH boom
 
-panasonic.chr:	../../dist/dk/db/digi-key.equ panasonic.gen
+panasonic.chr:	../../dist/dk/db/all.equ panasonic.gen
 	$(BOOM) gen2chr PANASONIC $^ >$@ || { rm -rf $@; exit 1; }

boom/manu/stackpole/Makefile

@@ -1,4 +1,4 @@
 BOOM=PATH=/home/moko/svn.openmoko.org/trunk/eda/boom:../boom:$$PATH boom
 
-stackpole.chr:	../../dist/dk/db/digi-key.equ stackpole.gen
+stackpole.chr:	../../dist/dk/db/all.equ stackpole.gen
 	$(BOOM) gen2chr STACKPOLE $^ >$@ || { rm -rf $@; exit 1; }

boom/manu/yageo/Makefile

@@ -1,4 +1,4 @@
 BOOM=PATH=/home/moko/svn.openmoko.org/trunk/eda/boom:../boom:$$PATH boom
 
-yageo.chr:	../../dist/dk/db/digi-key.equ yageo.gen
+yageo.chr:	../../dist/dk/db/all.equ yageo.gen
 	$(BOOM) gen2chr YAGEO $^ >$@ || { rm -rf $@; exit 1; }