Remove Node.sig and Node.cache_sig

This commit is contained in:
Thomas Nagy 2015-12-23 17:50:48 +01:00
parent db31168eb6
commit 7681bddfb7
19 changed files with 42 additions and 541 deletions

2
TODO
View File

@ -6,7 +6,6 @@ Waf 1.9
* Ensure _cache.py are valid python files
* Set cflags in the beginning / cppflags at the end
* Rework qt5
* Remove Node.cache_sig
* Make lazy visual studio detection the default
* Do not cache waf tool detection
* Fix the vala detection
@ -24,4 +23,5 @@ Done
* Remove qt4 and kde4 from the default modules
* Detect Clang first on many platforms, in particular on FreeBSD
* Let run_once accept a list of *args
* Remove Node.cache_sig and Node.sig

View File

@ -77,6 +77,3 @@ def dynamic_post(self):
self.source = Utils.to_list(self.source)
self.source.extend(self.path.get_bld().ant_glob(self.dynamic_source, remove=False))
# if headers are created dynamically, assign signatures manually:
# for x in self.path.get_bld().ant_glob('**/*.h', remove=False): x.sig = Utils.h_file(x.abspath())

View File

@ -21,7 +21,7 @@ def configure(conf):
def read_files(task):
# the real build files must be excluded, else they will get rebuilt
for x in task.generator.bld.bldnode.ant_glob('**', excl='**/*.o app', remove=False):
x.sig = Utils.h_file(x.abspath())
pass
def build(bld):
bld.post_mode = Build.POST_LAZY

View File

@ -129,7 +129,6 @@ def build(bld):
if x.endswith('.java'):
# create a node in the directory we want to
j = node.make_node(x) # create a node
j.sig = Utils.h_file(j.abspath()) # update the node signature
# depend on the .i file to make sure the .java files are copied after swig is executed
bld(name='move_and_read', rule=move_java_files, source='extend/java/test_swig_waf.i', after=['swig'], before=['javac'])
#"""

View File

@ -218,11 +218,9 @@ class BuildContext(Context.Context):
for f in env[CFG_FILES]:
newnode = self.root.find_resource(f)
try:
h = Utils.h_file(newnode.abspath())
except (IOError, AttributeError):
Logs.error('cannot find %r' % f)
h = Utils.SIG_NIL
newnode.sig = h
Utils.h_file(newnode.abspath())
except (IOError, OSError, AttributeError):
raise ValueError('Missing configuration file %r, reconfigure the project' % f)
def init_dirs(self):
"""

View File

@ -20,7 +20,7 @@ WAFVERSION="1.8.17"
WAFREVISION="cd7579a727d1b390bf9cbf111c1b20e811370bc0"
"""Git revision when the waf version is updated"""
ABI = 98
ABI = 99
"""Version of the build data cache file format (used in :py:const:`waflib.Context.DBFILE`)"""
DBFILE = '.wafpickle-%s-%d-%d' % (sys.platform, sys.hexversion, ABI)

View File

@ -69,7 +69,7 @@ class Node(object):
The Node objects are not thread safe in any way.
"""
dict_class = dict
__slots__ = ('name', 'sig', 'children', 'parent', 'cache_abspath', 'cache_isdir')
__slots__ = ('name', 'parent', 'children', 'cache_abspath', 'cache_isdir')
def __init__(self, name, parent):
self.name = name
self.parent = parent
@ -86,12 +86,10 @@ class Node(object):
if data[2] is not None:
# Issue 1480
self.children = self.dict_class(data[2])
if data[3] is not None:
self.sig = data[3]
def __getstate__(self):
"Serialize the node info"
return (self.name, self.parent, getattr(self, 'children', None), getattr(self, 'sig', None))
return (self.name, self.parent, getattr(self, 'children', None))
def __str__(self):
"String representation (name), for debugging purposes"
@ -745,14 +743,12 @@ class Node(object):
node = self.get_bld().search_node(lst)
if node:
if not os.path.isfile(node.abspath()):
node.sig = None
node.parent.mkdir()
return node
self = self.get_src()
node = self.find_node(lst)
if node:
if not os.path.isfile(node.abspath()):
node.sig = None
node.parent.mkdir()
return node
node = self.get_bld().make_node(lst)
@ -829,21 +825,15 @@ class Node(object):
the signature calculation relies on an existing attribute. Else the
signature is calculated automatically.
"""
if not self.is_bld() or self.ctx.bldnode is self.ctx.srcnode:
return self.h_file()
try:
return self.sig
except AttributeError:
return None
# previous behaviour can be set by returning self.ctx.task_sigs[self.abspath()] when a build node
return self.h_file()
# --------------------------------------------
# TODO waf 2.0, remove the cache_sig attribute
def get_cache_sig(self):
return self.sig
def set_cache_sig(self, v):
self.sig = v
cache_sig = property(get_cache_sig, set_cache_sig)
def get_sig(self):
return self.h_file()
sig = property(get_sig, Utils.nada)
cache_sig = property(get_sig, Utils.nada)
pickle_lock = Utils.threading.Lock()
"""Lock mandatory for thread-safe node serialization"""

View File

@ -591,8 +591,6 @@ class Task(TaskBase):
if not t.hasrun:
return ASK_LATER
bld = self.generator.bld
# first compute the signature
try:
new_sig = self.signature()
@ -600,24 +598,32 @@ class Task(TaskBase):
return ASK_LATER
# compare the signature to a signature computed previously
bld = self.generator.bld
key = self.uid()
try:
prev_sig = bld.task_sigs[key]
except KeyError:
Logs.debug("task: task %r must run as it was never run before or the task code changed" % self)
Logs.debug("task: task %r must run: it was never run before or the task code changed" % self)
return RUN_ME
if new_sig != prev_sig:
Logs.debug("task: task %r must run: the task signature changed" % self)
return RUN_ME
# compare the signatures of the outputs
for node in self.outputs:
try:
if node.sig != new_sig:
return RUN_ME
except AttributeError:
Logs.debug("task: task %r must run as the output nodes do not exist" % self)
p = node.abspath()
sig = bld.task_sigs.get(p, None)
if not sig:
Logs.debug("task: task %r must run: an output node has no signature" % self)
return RUN_ME
if sig != key:
Logs.debug("task: task %r must run: an output node was produced by another task" % self)
return RUN_ME
if not os.path.exists(p):
Logs.debug("task: task %r must run: an output node does not exist" % self)
return RUN_ME
if new_sig != prev_sig:
return RUN_ME
return self.never_skip or SKIP_ME
def post_run(self):
@ -630,7 +636,6 @@ class Task(TaskBase):
"""
bld = self.generator.bld
sig = self.signature()
for node in self.outputs:
# check if the node exists
try:
@ -641,7 +646,7 @@ class Task(TaskBase):
raise Errors.WafError(self.err_msg)
# important, store the signature for the next run
node.sig = sig
bld.task_sigs[node.abspath()] = self.uid() # make sure this task produced the files in question
bld.task_sigs[self.uid()] = self.cache_sig
@ -1154,56 +1159,7 @@ def always_run(cls):
def update_outputs(cls):
"""
Task class decorator
If you want to create files in the source directory. For example, to keep *foo.txt* in the source
directory, create it first and declare::
def build(bld):
bld(rule='cp ${SRC} ${TGT}', source='wscript', target='foo.txt', update_outputs=True)
Obsolete, will be removed in waf 2.0
"""
old_post_run = cls.post_run
def post_run(self):
old_post_run(self)
for node in self.outputs:
node.sig = Utils.h_file(node.abspath())
self.generator.bld.task_sigs[node.abspath()] = self.uid() # issue #1017
cls.post_run = post_run
old_runnable_status = cls.runnable_status
def runnable_status(self):
status = old_runnable_status(self)
if status != RUN_ME:
return status
try:
# by default, we check that the output nodes have the signature of the task
# perform a second check, returning 'SKIP_ME' as we are expecting that
# the signatures do not match
bld = self.generator.bld
prev_sig = bld.task_sigs[self.uid()]
if prev_sig == self.signature():
for x in self.outputs:
if not x.is_child_of(bld.bldnode):
# special case of files created in the source directory
# hash them here for convenience -_-
x.sig = Utils.h_file(x.abspath())
if not x.sig or bld.task_sigs[x.abspath()] != self.uid():
return RUN_ME
return SKIP_ME
except OSError:
pass
except IOError:
pass
except KeyError:
pass
except IndexError:
pass
except AttributeError:
pass
return RUN_ME
cls.runnable_status = runnable_status
return cls

View File

@ -820,7 +820,6 @@ def process_subst(self):
a = self.path.find_node(x)
b = self.path.get_bld().make_node(y)
if not os.path.isfile(b.abspath()):
b.sig = None
b.parent.mkdir()
else:
if isinstance(x, str):

View File

@ -600,9 +600,6 @@ class fake_shlib(link_task):
for t in self.run_after:
if not t.hasrun:
return Task.ASK_LATER
for x in self.outputs:
x.sig = Utils.h_file(x.abspath())
return Task.SKIP_ME
class fake_stlib(stlink_task):
@ -613,9 +610,6 @@ class fake_stlib(stlink_task):
for t in self.run_after:
if not t.hasrun:
return Task.ASK_LATER
for x in self.outputs:
x.sig = Utils.h_file(x.abspath())
return Task.SKIP_ME
@conf
@ -658,7 +652,10 @@ def process_lib(self):
for y in names:
node = x.find_node(y)
if node:
node.sig = Utils.h_file(node.abspath())
try:
Utils.h_file(node.abspath())
except (IOError, OSError):
raise ValueError('Could not read %r' % y)
break
else:
continue

View File

@ -198,8 +198,6 @@ class fake_csshlib(Task.Task):
inst_to = None
def runnable_status(self):
for x in self.outputs:
x.sig = Utils.h_file(x.abspath())
return Task.SKIP_ME
@conf

View File

@ -111,8 +111,6 @@ class fc(Task.Task):
if x.startswith('MOD@'):
name = bld.modfile(x.replace('MOD@', ''))
node = bld.srcnode.find_or_declare(name)
if not getattr(node, 'sig', None):
node.sig = Utils.SIG_NIL
tsk.set_outputs(node)
outs[id(node)].add(tsk)

View File

@ -300,8 +300,8 @@ class javac(Task.Task):
def post_run(self):
"""
"""
for n in self.generator.outdir.ant_glob('**/*.class'):
n.sig = Utils.h_file(n.abspath()) # careful with this
for node in self.generator.outdir.ant_glob('**/*.class'):
bld.task_sigs[node.abspath()] = self.uid()
self.generator.bld.task_sigs[self.uid()] = self.cache_sig
@feature('javadoc')
@ -350,8 +350,8 @@ class javadoc(Task.Task):
def post_run(self):
nodes = self.generator.javadoc_output.ant_glob('**')
for x in nodes:
x.sig = Utils.h_file(x.abspath())
for nodes in nodes:
bld.task_sigs[node.abspath()] = self.uid()
self.generator.bld.task_sigs[self.uid()] = self.cache_sig
def configure(self):

View File

@ -148,7 +148,7 @@ class doxygen(Task.Task):
def post_run(self):
nodes = self.output_dir.ant_glob('**/*', quiet=True)
for x in nodes:
x.sig = Utils.h_file(x.abspath())
self.generator.bld.task_sigs[x.abspath()] = self.uid()
self.add_install()
return Task.Task.post_run(self)

View File

@ -64,8 +64,6 @@ class cgopackage(stlink_task):
b.write(s.read())
bld_srcs.append(b)
#print("--|> [%s]" % b.abspath())
b.sig = Utils.h_file(b.abspath())
pass
#self.set_inputs(bld_srcs)
#self.generator.bld.raw_deps[self.uid()] = [self.signature()] + bld_srcs
makefile_node = bld_dir.make_node("Makefile")

View File

@ -1,411 +0,0 @@
#!/usr/bin/env python
# encoding: utf-8
# Thomas Nagy, 2006-2010 (ita)
"""
This tool is totally deprecated
Try using:
.pc.in files for .pc files
the feature intltool_in - see demos/intltool
make-like rules
"""
import shutil, re, os
from waflib import Node, Task, Utils, Errors
from waflib.TaskGen import feature, after_method, before_method
from waflib.Logs import debug
def copy_attrs(orig, dest, names, only_if_set=False):
"""
copy class attributes from an object to another
"""
for a in Utils.to_list(names):
u = getattr(orig, a, ())
if u or not only_if_set:
setattr(dest, a, u)
def copy_func(tsk):
"Make a file copy. This might be used to make other kinds of file processing (even calling a compiler is possible)"
infile = tsk.inputs[0].abspath()
outfile = tsk.outputs[0].abspath()
try:
shutil.copy2(infile, outfile)
except EnvironmentError:
return 1
else:
if tsk.chmod: os.chmod(outfile, tsk.chmod)
return 0
def action_process_file_func(tsk):
"Ask the function attached to the task to process it"
if not tsk.fun: raise Errors.WafError('task must have a function attached to it for copy_func to work!')
return tsk.fun(tsk)
@feature('cmd')
def apply_cmd(self):
"call a command everytime"
if not self.fun: raise Errors.WafError('cmdobj needs a function!')
tsk = Task.TaskBase()
tsk.fun = self.fun
tsk.env = self.env
self.tasks.append(tsk)
tsk.install_path = self.install_path
@feature('copy')
@before_method('process_source')
def apply_copy(self):
Utils.def_attrs(self, fun=copy_func)
self.default_install_path = 0
lst = self.to_list(self.source)
self.meths.remove('process_source')
for filename in lst:
node = self.path.find_resource(filename)
if not node: raise Errors.WafError('cannot find input file %s for processing' % filename)
target = self.target
if not target or len(lst)>1: target = node.name
# TODO the file path may be incorrect
newnode = self.path.find_or_declare(target)
tsk = self.create_task('copy', node, newnode)
tsk.fun = self.fun
tsk.chmod = getattr(self, 'chmod', Utils.O644)
if not tsk.env:
tsk.debug()
raise Errors.WafError('task without an environment')
def subst_func(tsk):
"Substitutes variables in a .in file"
m4_re = re.compile('@(\w+)@', re.M)
code = tsk.inputs[0].read() #Utils.readf(infile)
# replace all % by %% to prevent errors by % signs in the input file while string formatting
code = code.replace('%', '%%')
s = m4_re.sub(r'%(\1)s', code)
env = tsk.env
di = getattr(tsk, 'dict', {}) or getattr(tsk.generator, 'dict', {})
if not di:
names = m4_re.findall(code)
for i in names:
di[i] = env.get_flat(i) or env.get_flat(i.upper())
tsk.outputs[0].write(s % di)
@feature('subst')
@before_method('process_source')
def apply_subst(self):
Utils.def_attrs(self, fun=subst_func)
lst = self.to_list(self.source)
self.meths.remove('process_source')
self.dict = getattr(self, 'dict', {})
for filename in lst:
node = self.path.find_resource(filename)
if not node: raise Errors.WafError('cannot find input file %s for processing' % filename)
if self.target:
newnode = self.path.find_or_declare(self.target)
else:
newnode = node.change_ext('')
try:
self.dict = self.dict.get_merged_dict()
except AttributeError:
pass
if self.dict and not self.env['DICT_HASH']:
self.env = self.env.derive()
keys = list(self.dict.keys())
keys.sort()
lst = [self.dict[x] for x in keys]
self.env['DICT_HASH'] = str(Utils.h_list(lst))
tsk = self.create_task('copy', node, newnode)
tsk.fun = self.fun
tsk.dict = self.dict
tsk.dep_vars = ['DICT_HASH']
tsk.chmod = getattr(self, 'chmod', Utils.O644)
if not tsk.env:
tsk.debug()
raise Errors.WafError('task without an environment')
####################
## command-output ####
####################
class cmd_arg(object):
"""command-output arguments for representing files or folders"""
def __init__(self, name, template='%s'):
self.name = name
self.template = template
self.node = None
class input_file(cmd_arg):
def find_node(self, base_path):
assert isinstance(base_path, Node.Node)
self.node = base_path.find_resource(self.name)
if self.node is None:
raise Errors.WafError("Input file %s not found in " % (self.name, base_path))
def get_path(self, env, absolute):
if absolute:
return self.template % self.node.abspath()
else:
return self.template % self.node.srcpath()
class output_file(cmd_arg):
def find_node(self, base_path):
assert isinstance(base_path, Node.Node)
self.node = base_path.find_or_declare(self.name)
if self.node is None:
raise Errors.WafError("Output file %s not found in " % (self.name, base_path))
def get_path(self, env, absolute):
if absolute:
return self.template % self.node.abspath()
else:
return self.template % self.node.bldpath()
class cmd_dir_arg(cmd_arg):
def find_node(self, base_path):
assert isinstance(base_path, Node.Node)
self.node = base_path.find_dir(self.name)
if self.node is None:
raise Errors.WafError("Directory %s not found in " % (self.name, base_path))
class input_dir(cmd_dir_arg):
def get_path(self, dummy_env, dummy_absolute):
return self.template % self.node.abspath()
class output_dir(cmd_dir_arg):
def get_path(self, env, dummy_absolute):
return self.template % self.node.abspath()
class command_output(Task.Task):
color = "BLUE"
def __init__(self, env, command, command_node, command_args, stdin, stdout, cwd, os_env, stderr):
Task.Task.__init__(self, env=env)
assert isinstance(command, (str, Node.Node))
self.command = command
self.command_args = command_args
self.stdin = stdin
self.stdout = stdout
self.cwd = cwd
self.os_env = os_env
self.stderr = stderr
if command_node is not None: self.dep_nodes = [command_node]
self.dep_vars = [] # additional environment variables to look
def run(self):
task = self
#assert len(task.inputs) > 0
def input_path(node, template):
if task.cwd is None:
return template % node.bldpath()
else:
return template % node.abspath()
def output_path(node, template):
fun = node.abspath
if task.cwd is None: fun = node.bldpath
return template % fun()
if isinstance(task.command, Node.Node):
argv = [input_path(task.command, '%s')]
else:
argv = [task.command]
for arg in task.command_args:
if isinstance(arg, str):
argv.append(arg)
else:
assert isinstance(arg, cmd_arg)
argv.append(arg.get_path(task.env, (task.cwd is not None)))
if task.stdin:
stdin = open(input_path(task.stdin, '%s'))
else:
stdin = None
if task.stdout:
stdout = open(output_path(task.stdout, '%s'), "w")
else:
stdout = None
if task.stderr:
stderr = open(output_path(task.stderr, '%s'), "w")
else:
stderr = None
if task.cwd is None:
cwd = ('None (actually %r)' % os.getcwd())
else:
cwd = repr(task.cwd)
debug("command-output: cwd=%s, stdin=%r, stdout=%r, argv=%r" %
(cwd, stdin, stdout, argv))
if task.os_env is None:
os_env = os.environ
else:
os_env = task.os_env
command = Utils.subprocess.Popen(argv, stdin=stdin, stdout=stdout, stderr=stderr, cwd=task.cwd, env=os_env)
return command.wait()
@feature('command-output')
def init_cmd_output(self):
Utils.def_attrs(self,
stdin = None,
stdout = None,
stderr = None,
# the command to execute
command = None,
# whether it is an external command; otherwise it is assumed
# to be an executable binary or script that lives in the
# source or build tree.
command_is_external = False,
# extra parameters (argv) to pass to the command (excluding
# the command itself)
argv = [],
# dependencies to other objects -> this is probably not what you want (ita)
# values must be 'task_gen' instances (not names!)
dependencies = [],
# dependencies on env variable contents
dep_vars = [],
# input files that are implicit, i.e. they are not
# stdin, nor are they mentioned explicitly in argv
hidden_inputs = [],
# output files that are implicit, i.e. they are not
# stdout, nor are they mentioned explicitly in argv
hidden_outputs = [],
# change the subprocess to this cwd (must use obj.input_dir() or output_dir() here)
cwd = None,
# OS environment variables to pass to the subprocess
# if None, use the default environment variables unchanged
os_env = None)
@feature('command-output')
@after_method('init_cmd_output')
def apply_cmd_output(self):
if self.command is None:
raise Errors.WafError("command-output missing command")
if self.command_is_external:
cmd = self.command
cmd_node = None
else:
cmd_node = self.path.find_resource(self.command)
assert cmd_node is not None, ('''Could not find command '%s' in source tree.
Hint: if this is an external command,
use command_is_external=True''') % (self.command,)
cmd = cmd_node
if self.cwd is None:
cwd = None
inputs = []
outputs = []
for arg in self.argv:
if isinstance(arg, cmd_arg):
arg.find_node(self.path)
if isinstance(arg, input_file):
inputs.append(arg.node)
if isinstance(arg, output_file):
outputs.append(arg.node)
if self.stdout is None:
stdout = None
else:
assert isinstance(self.stdout, str)
stdout = self.path.find_or_declare(self.stdout)
if stdout is None:
raise Errors.WafError("File %s not found" % (self.stdout,))
outputs.append(stdout)
if self.stderr is None:
stderr = None
else:
assert isinstance(self.stderr, str)
stderr = self.path.find_or_declare(self.stderr)
if stderr is None:
raise Errors.WafError("File %s not found" % (self.stderr,))
outputs.append(stderr)
if self.stdin is None:
stdin = None
else:
assert isinstance(self.stdin, str)
stdin = self.path.find_resource(self.stdin)
if stdin is None:
raise Errors.WafError("File %s not found" % (self.stdin,))
inputs.append(stdin)
for hidden_input in self.to_list(self.hidden_inputs):
node = self.path.find_resource(hidden_input)
if node is None:
raise Errors.WafError("File %s not found in dir %s" % (hidden_input, self.path))
inputs.append(node)
for hidden_output in self.to_list(self.hidden_outputs):
node = self.path.find_or_declare(hidden_output)
if node is None:
raise Errors.WafError("File %s not found in dir %s" % (hidden_output, self.path))
outputs.append(node)
if not (inputs or getattr(self, 'no_inputs', None)):
raise Errors.WafError('command-output objects must have at least one input file or give self.no_inputs')
if not (outputs or getattr(self, 'no_outputs', None)):
raise Errors.WafError('command-output objects must have at least one output file or give self.no_outputs')
cwd = self.bld.variant_dir
task = command_output(self.env, cmd, cmd_node, self.argv, stdin, stdout, cwd, self.os_env, stderr)
task.generator = self
copy_attrs(self, task, 'before after ext_in ext_out', only_if_set=True)
self.tasks.append(task)
task.inputs = inputs
task.outputs = outputs
task.dep_vars = self.to_list(self.dep_vars)
for dep in self.dependencies:
assert dep is not self
dep.post()
for dep_task in dep.tasks:
task.set_run_after(dep_task)
if not task.inputs:
# the case for svnversion, always run, and update the output nodes
task.runnable_status = type(Task.TaskBase.run)(runnable_status, task, task.__class__) # always run
task.post_run = type(Task.TaskBase.run)(post_run, task, task.__class__)
# TODO the case with no outputs?
def post_run(self):
for x in self.outputs:
x.sig = Utils.h_file(x.abspath())
def runnable_status(self):
return self.RUN_ME
Task.task_factory('copy', vars=[], func=action_process_file_func)

View File

@ -230,11 +230,6 @@ def can_retrieve_cache(self):
if err:
return False
for node in self.outputs:
node.sig = sig
#if self.generator.bld.progress_bar < 1:
# self.generator.bld.to_log('restoring from cache %r\n' % node.abspath())
self.cached = True
return True

View File

@ -17,10 +17,6 @@ and output files. While a typical script may require the following::
k.cache_isdir = True
k = k.parent
# clear the file if removed
if not os.path.isfile(node.abspath()):
node.sig = None
# create the folder structure
if node.parent.height() > 2:
node.parent.mkdir()
@ -83,7 +79,6 @@ def find_or_declare(self, lst):
if not ret:
ret = node.make_node(lst[2:])
if not os.path.isfile(ret.abspath()):
ret.sig = None
ret.parent.mkdir()
return ret

View File

@ -99,13 +99,7 @@ if Utils.is_win32:
pass
except AttributeError:
self.ctx.hash_cache = {}
if not self.is_bld():
if self.is_child_of(self.ctx.srcnode):
self.sig = self.cached_hash_file()
else:
self.sig = Utils.h_file(self.abspath())
self.ctx.hash_cache[id(self)] = ret = self.sig
self.ctx.hash_cache[id(self)] = ret = Utils.h_file(self.abspath())
return ret
Node.Node.get_bld_sig = get_bld_sig_win32
@ -153,7 +147,6 @@ if Utils.is_win32:
node = self.get_bld().search(lst)
if node:
if not node.isfile_cached():
node.sig = None
try:
node.parent.mkdir()
except OSError:
@ -163,7 +156,6 @@ if Utils.is_win32:
node = self.find_node(lst)
if node:
if not node.isfile_cached():
node.sig = None
try:
node.parent.mkdir()
except OSError: