diff --git a/playground/weak_constraints/greenfirst.py b/playground/weak_constraints/greenfirst.py new file mode 100644 index 00000000..50645905 --- /dev/null +++ b/playground/weak_constraints/greenfirst.py @@ -0,0 +1,63 @@ +#! /usr/bin/env python +# encoding: utf-8 + +from waflib import Task, Runner + +# amount of tasks to wait before trying to mark tasks as done +# see Runner.py for details +# setting the gap to a low value may seriously degrade performance +Runner.GAP = 1 + +# shrinking sets of dependencies provided to know quickly which yelow tasks are ready +reverse_map = {} + +old = Task.set_file_constraints +def green_first(lst): + # weak order constraint based on lexicographic order: + # green before pink before yellow + lst.sort(cmp=lambda x, y: cmp(x.__class__.__name__, y.__class__.__name__)) + + # call the previous method to order the tasks by file dependencies + old(lst) + + # shrinking sets preparation + for tsk in lst: + for k in tsk.run_after: + try: + reverse_map[k].add(tsk) + except KeyError: + reverse_map[k] = set([tsk]) +Task.set_file_constraints = green_first + +def get_out(self): + # this is called whenever a task has finished executing + tsk = self.prev_get_out() + + for x in reverse_map.get(tsk, []): + # remote this task from the dependencies of other tasks + # this is a minor optimization in general + # but here we use this to know which tasks are ready to run + x.run_after.remove(tsk) + + if tsk.__class__.__name__ == 'green': + # whenever a green task is done it may be time to put + # one or more yellow tasks in front + # some additional optimization can be performed if exactly one + # yellow task can be added to avoid whole list traversal + def extract_yellow(lst): + # return the yellow tasks that we can run immediately + front = [] + lst.reverse() + for tsk in lst: + if tsk.__class__.__name__ == 'yellow' and not tsk.run_after: + #print("found one yellow task to run first") + lst.remove(tsk) + front.append(tsk) + lst.reverse() + return front + # this sets the order again + self.outstanding = extract_yellow(self.outstanding) + extract_yellow(self.frozen) + self.outstanding + return tsk +Runner.Parallel.prev_get_out = Runner.Parallel.get_out +Runner.Parallel.get_out = get_out + diff --git a/playground/weak_constraints/wscript b/playground/weak_constraints/wscript new file mode 100644 index 00000000..b0921892 --- /dev/null +++ b/playground/weak_constraints/wscript @@ -0,0 +1,40 @@ +#! /usr/bin/env python +# encoding: utf-8 + +""" +Illustrates how to force yellow tasks to be executed very soon after the green ones +""" + +import time + +def options(ctx): + ctx.load('parallel_debug') + +def configure(ctx): + ctx.load('parallel_debug') + +def build(ctx): + + ctx.load('greenfirst', tooldir='.') + + ctx.jobs = 4 + def touch(tsk): + tsk.outputs[0].write('') + if tsk.__class__.__name__ == 'green': + time.sleep(0.2) + elif tsk.__class__.__name__ == 'yellow': + time.sleep(3) + else: + time.sleep(0.3) + + for x in range(40): + ctx(rule=touch, always=True, name='pink', color='PINK', target='pink%s.txt' % x) + + for x in range(5): + lst = [] + for y in range(60): + name = 'green%s_%s.txt' % (x, y) + lst.append(name) + ctx(rule=touch, always=True, target=name, name='green', color='GREEN') + ctx(rule=touch, always=True, source=lst, name='yellow', color='YELLOW', target='yellow%s.txt' % x) +