clang_compilation_database: fix #2247, add clangdb command to generate database by request without rebuilding, add tests (WIP)

playground/clang_compilation_database/wscript

@@ -0,0 +1,68 @@
+#! /usr/bin/env python
+# encoding: utf-8
+# Alibek Omarov, 2019 (a1batross)
+
+import os
+from waflib import ConfigSet, Logs
+
+VERSION='0.0.1'
+APPNAME='clang_compilation_database_test'
+
+top = '.'
+out = 'build'
+
+INCLUDES_TO_TEST = ['common'] # check if include flag appeared in result json
+DEFINES_TO_TEST = ['TEST'] # check if definition flag will appear in result json
+SOURCE_FILES_TO_TEST = ['a.c', 'b.cpp'] # check if source files are persist in database
+
+def actual_test(bld):
+	db = bld.bldnode.find_node('compile_commands.json').read_json()
+
+	for entry in db:
+		env = ConfigSet.ConfigSet()
+		line = ' '.join(entry['arguments'][1:]) # ignore compiler exe, unneeded
+		directory = entry['directory']
+		srcname = entry['file'].split(os.sep)[-1] # file name only
+
+		bld.parse_flags(line, 'test', env) # ignore unhandled flag, it's harmless for test
+
+		if bld.bldnode.abspath() in directory:
+			Logs.info('Directory test passed')
+		else:
+			Logs.error('Directory test failed')
+
+		if srcname in SOURCE_FILES_TO_TEST:
+			Logs.info('Source file test passed')
+		else:
+			Logs.error('Source file test failed')
+
+		passed = True
+		for inc in INCLUDES_TO_TEST:
+			if inc not in env.INCLUDES_test:
+				passed = False
+
+		if passed: Logs.info('Includes test passed')
+		else: Logs.error('Includes test failed')
+
+		passed = True
+		for define in DEFINES_TO_TEST:
+			if define not in env.DEFINES_test:
+				passed = False
+		if passed: Logs.info('Defines test passed')
+		else: Logs.error('Defines test failed')
+
+def options(opt):
+	# check by ./waf clangdb
+	opt.load('compiler_c compiler_cxx clang_compilation_database')
+
+def configure(conf):
+	# check if database always generated before build
+	conf.load('compiler_c compiler_cxx clang_compilation_database')
+
+def build(bld):
+	bld.shlib(features = 'c cxx', source = SOURCE_FILES_TO_TEST,
+		defines = DEFINES_TO_TEST,
+		includes = INCLUDES_TO_TEST,
+		target = 'test')
+
+	bld.add_post_fun(actual_test)

waflib/extras/clang_compilation_database.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # encoding: utf-8
 # Christoph Koke, 2013
+# Alibek Omarov, 2019
 
 """
 Writes the c and cpp compile commands into build/compile_commands.json
@@ -8,14 +9,24 @@ see http://clang.llvm.org/docs/JSONCompilationDatabase.html
 
 Usage:
 
-    def configure(conf):
-        conf.load('compiler_cxx')
-        ...
-        conf.load('clang_compilation_database')
+	Load this tool in `options` to be able to generate database
+	by request in command-line and before build:
+
+	$ waf clangdb
+
+	def options(opt):
+		opt.load('clang_compilation_database')
+
+	Otherwise, load only in `configure` to generate it always before build.
+
+	def configure(conf):
+		conf.load('compiler_cxx')
+		...
+		conf.load('clang_compilation_database')
 """
 
-import sys, os, json, shlex, pipes
-from waflib import Logs, TaskGen, Task
+import os
+from waflib import Logs, TaskGen, Task, Build, Scripting
 
 Task.Task.keep_last_cmd = True
 
@@ -23,63 +34,104 @@ Task.Task.keep_last_cmd = True
 @TaskGen.after_method('process_use')
 def collect_compilation_db_tasks(self):
 	"Add a compilation database entry for compiled tasks"
-	try:
-		clang_db = self.bld.clang_compilation_database_tasks
-	except AttributeError:
-		clang_db = self.bld.clang_compilation_database_tasks = []
-		self.bld.add_post_fun(write_compilation_database)
+	if not isinstance(self.bld, ClangDbContext):
+		return
 
 	tup = tuple(y for y in [Task.classes.get(x) for x in ('c', 'cxx')] if y)
 	for task in getattr(self, 'compiled_tasks', []):
 		if isinstance(task, tup):
-			clang_db.append(task)
+			self.bld.clang_compilation_database_tasks.append(task)
 
-def write_compilation_database(ctx):
-	"Write the clang compilation database as JSON"
-	database_file = ctx.bldnode.make_node('compile_commands.json')
-	Logs.info('Build commands will be stored in %s', database_file.path_from(ctx.path))
-	try:
-		root = json.load(database_file)
-	except IOError:
-		root = []
-	clang_db = dict((x['file'], x) for x in root)
-	for task in getattr(ctx, 'clang_compilation_database_tasks', []):
+class ClangDbContext(Build.BuildContext):
+	'''generates compile_commands.json by request'''
+	cmd = 'clangdb'
+	clang_compilation_database_tasks = []
+	
+	def write_compilation_database(self):
+		"""
+		Write the clang compilation database as JSON
+		"""
+		database_file = self.bldnode.make_node('compile_commands.json')
+		Logs.info('Build commands will be stored in %s', database_file.path_from(self.path))
 		try:
-			cmd = task.last_cmd
-		except AttributeError:
-			continue
-		directory = getattr(task, 'cwd', ctx.variant_dir)
-		f_node = task.inputs[0]
-		filename = os.path.relpath(f_node.abspath(), directory)
-		entry = {
-			"directory": directory,
-			"arguments": cmd,
-			"file": filename,
-		}
-		clang_db[filename] = entry
-	root = list(clang_db.values())
-	database_file.write(json.dumps(root, indent=2))
+			root = database_file.read_json()
+		except IOError:
+			root = []
+		clang_db = dict((x['file'], x) for x in root)
+		for task in self.clang_compilation_database_tasks:
+			try:
+				cmd = task.last_cmd
+			except AttributeError:
+				continue
+			directory = getattr(task, 'cwd', self.variant_dir)
+			f_node = task.inputs[0]
+			filename = os.path.relpath(f_node.abspath(), directory)
+			entry = {
+				"directory": directory,
+				"arguments": cmd,
+				"file": filename,
+			}
+			clang_db[filename] = entry
+		root = list(clang_db.values())
+		database_file.write_json(root)
 
-# Override the runnable_status function to do a dummy/dry run when the file doesn't need to be compiled.
-# This will make sure compile_commands.json is always fully up to date.
-# Previously you could end up with a partial compile_commands.json if the build failed.
-for x in ('c', 'cxx'):
-	if x not in Task.classes:
-		continue
+	def execute(self):
+		"""
+		Build dry run
+		"""
+		self.restore()
 
-	t = Task.classes[x]
+		if not self.all_envs:
+			self.load_envs()
 
-	def runnable_status(self):
-		def exec_command(cmd, **kw):
-			pass
+		self.recurse([self.run_dir])
+		self.pre_build()
 
-		run_status = self.old_runnable_status()
-		if run_status == Task.SKIP_ME:
-			setattr(self, 'old_exec_command', getattr(self, 'exec_command', None))
-			setattr(self, 'exec_command', exec_command)
-			self.run()
-			setattr(self, 'exec_command', getattr(self, 'old_exec_command', None))
-		return run_status
+		# we need only to generate last_cmd, so override
+		# exec_command temporarily
+		def exec_command(self, *k, **kw):
+			return 0
+		
+		for g in self.groups:
+			for tg in g:
+				try:
+					f = tg.post
+				except AttributeError:
+					pass
+				else:
+					f()
+				
+				if isinstance(tg, Task.Task):
+					lst = [tg]
+				else: lst = tg.tasks
+				for tsk in lst:
+					tup = tuple(y for y in [Task.classes.get(x) for x in ('c', 'cxx')] if y)
+					if isinstance(tsk, tup):
+						old_exec = tsk.exec_command
+						tsk.exec_command = exec_command
+						tsk.run()
+						tsk.exec_command = old_exec
 
-	setattr(t, 'old_runnable_status', getattr(t, 'runnable_status', None))
-	setattr(t, 'runnable_status', runnable_status)
+		self.write_compilation_database()
+
+EXECUTE_PATCHED = False
+def patch_execute():
+	global EXECUTE_PATCHED
+	
+	if EXECUTE_PATCHED:
+		return
+
+	def new_execute_build(self):
+		"""
+		Invoke clangdb command before build
+		"""
+		if type(self) == Build.BuildContext:
+			Scripting.run_command('clangdb')
+			
+		old_execute_build(self)
+
+	old_execute_build = getattr(Build.BuildContext, 'execute_build', None)
+	setattr(Build.BuildContext, 'execute_build', new_execute_build)
+	EXECUTE_PATCHED = True
+	
+patch_execute()