shrinking sets for the memory reducer

waflib/extras/mem_reducer.py

@@ -3,6 +3,7 @@
 
 """
 This tool can help to reduce the memory usage in very large builds featuring many tasks with after/before attributes.
+It may also improve the overall build time by decreasing the amount of iterations over tasks.
 
 Usage:
 def options(opt):
@@ -10,7 +11,7 @@ def options(opt):
 """
 
 import itertools
-from waflib import Utils, Task
+from waflib import Utils, Task, Runner
 
 class SetOfTasks(object):
 	"""Wraps a set and a task which has a list of other sets.
@@ -22,6 +23,7 @@ class SetOfTasks(object):
 
 	def __iter__(self):
 		for g in self._owner.run_after_groups:
+			#print len(g)
 			for task in g:
 				yield task
 		for task in self._set:
@@ -38,6 +40,7 @@ def set_precedence_constraints(tasks):
 	for x in tasks:
 		x.run_after = SetOfTasks(x)
 		x.run_after_groups = []
+		x.waiting_sets = []
 
 		h = x.hash_constraints()
 		cstr_groups[h].append(x)
@@ -58,9 +61,32 @@ def set_precedence_constraints(tasks):
 		if Task.is_before(t1, t2):
 			for x in group2:
 				x.run_after_groups.append(group1)
+			for k in group1:
+				k.waiting_sets.append(group1)
 		elif Task.is_before(t2, t1):
 			for x in group1:
 				x.run_after_groups.append(group2)
+			for k in group2:
+				k.waiting_sets.append(group2)
 
 Task.set_precedence_constraints = set_precedence_constraints
 
+def get_out(self):
+	tsk = self.out.get()
+	if not self.stop:
+		self.add_more_tasks(tsk)
+	self.count -= 1
+	self.dirty = True
+
+	# shrinking sets
+	try:
+		ws = tsk.waiting_sets
+	except AttributeError:
+		pass
+	else:
+		for k in ws:
+			k.remove(tsk)
+
+	return tsk
+Runner.Parallel.get_out = get_out
+