waf/waflib/extras/batched_cc.py

#!/usr/bin/env python
# encoding: utf-8
# Thomas Nagy, 2006-2010 (ita)

"""
Build as batches.

Instead of compiling object files one by one, c/c++ compilers are often able to compile at once:
cc -c ../file1.c ../file2.c ../file3.c

Files are output on the directory where the compiler is called, and dependencies are more difficult
to track (do not run the command on all source files if only one file changes)

As such, we do as if the files were compiled one by one, but no command is actually run:
replace each cc/cpp Task by a TaskSlave. A new task called TaskMaster collects the
signatures from each slave and finds out the command-line to run.

Just import this module in the configuration (no other change required).
This is provided as an example, for performance unity builds are recommended (fewer tasks and fewer jobs to execute).
"""

import os
from waflib import TaskGen, Task, Build, Logs
from waflib.TaskGen import extension, feature, before_method, after_method

MAX_BATCH = 50

c_str = '${CC} ${CFLAGS} ${CPPFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${CPPPATH_ST:INCPATHS} ${DEFINES_ST:DEFINES} -c ${SRCLST}'
#c_str = '${CC} ${CCFLAGS} ${CPPFLAGS} ${_CCINCFLAGS} ${_CCDEFFLAGS} -c ${SRCLST}'
c_fun, _ = Task.compile_fun_noshell(c_str)

cxx_str = '${CXX} ${CXXFLAGS} ${CPPFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${CPPPATH_ST:INCPATHS} ${DEFINES_ST:DEFINES} -c ${SRCLST}'
#cxx_str = '${CXX} ${CXXFLAGS} ${CPPFLAGS} ${_CXXINCFLAGS} ${_CXXDEFFLAGS} -c ${SRCLST}'
cxx_fun, _ = Task.compile_fun_noshell(cxx_str)

count = 70000
class batch_task(Task.Task):
	color = 'RED'

	after = ['c', 'cxx']
	before = ['cprogram', 'cshlib', 'cstlib', 'cxxprogram', 'cxxshlib', 'cxxstlib']

	def __str__(self):
		return '(batch compilation for %d slaves)\n' % len(self.slaves)

	def __init__(self, *k, **kw):
		Task.Task.__init__(self, *k, **kw)
		self.slaves = []
		self.inputs = []
		self.hasrun = 0

		global count
		count += 1
		self.idx = count

	def add_slave(self, slave):
		self.slaves.append(slave)
		self.set_run_after(slave)

	def runnable_status(self):
		for t in self.run_after:
			if not t.hasrun:
				return Task.ASK_LATER

		for t in self.slaves:
			#if t.executed:
			if t.hasrun != Task.SKIPPED:
				return Task.RUN_ME

		return Task.SKIP_ME

	def run(self):
		outputs = []
		self.outputs = []

		srclst = []
		slaves = []
		for t in self.slaves:
			if t.hasrun != Task.SKIPPED:
				slaves.append(t)
				srclst.append(t.inputs[0].abspath())

		self.env.SRCLST = srclst
		self.cwd = slaves[0].inputs[0].parent.get_bld().abspath()

		if self.slaves[0].__class__.__name__ == 'c':
			ret = c_fun(self)
		else:
			ret = cxx_fun(self)

		if ret:
			return ret

		for t in slaves:
			t.old_post_run()

from waflib.Tools import c, cxx

def hook(name):
	def n_hook(self, node):
		task = self.create_task(name, node, node.change_ext('.o'))
		try:
			self.compiled_tasks.append(task)
		except AttributeError:
			self.compiled_tasks = [task]

		if not getattr(self, 'masters', None):
			self.masters = {}
			self.allmasters = []

		if not node.parent in self.masters:
			m = self.masters[node.parent] = self.master = self.create_task('batch')
			self.allmasters.append(m)
		else:
			m = self.masters[node.parent]
			if len(m.slaves) > MAX_BATCH:
				m = self.masters[node.parent] = self.master = self.create_task('batch')
				self.allmasters.append(m)

		m.add_slave(task)
		return task
	return n_hook

extension('.c')(hook('c'))
extension('.cpp','.cc','.cxx','.C','.c++')(hook('cxx'))

@feature('cprogram', 'cshlib', 'cstaticlib', 'cxxprogram', 'cxxshlib', 'cxxstlib')
@after_method('apply_link')
def link_after_masters(self):
	if getattr(self, 'allmasters', None):
		for m in self.allmasters:
			self.link_task.set_run_after(m)

# Modify the c and cxx task classes - in theory it would be better to
# create subclasses and to re-map the c/c++ extensions
#
for x in ('c', 'cxx'):
	t = Task.classes[x]
	def run(self):
		pass

	def post_run(self):
		#self.executed=1
		pass

	setattr(t, 'oldrun', t.__dict__['run'])
	setattr(t, 'run', run)
	setattr(t, 'old_post_run', t.post_run)
	setattr(t, 'post_run', post_run)