From 5c6d626048c1613450b00eff23dd8beaddd4e5ef Mon Sep 17 00:00:00 2001 From: Thomas Nagy Date: Sun, 26 May 2013 10:14:49 +0200 Subject: [PATCH] removed the cache from the task level, it usually degrades performance and is better implemented on the compiler level --- ChangeLog | 2 +- docs/book/execution.txt | 1 - waflib/Build.py | 2 - waflib/Options.py | 3 - waflib/Task.py | 138 ------------------------------------ waflib/Tools/c_config.py | 2 - waflib/Tools/fc.py | 4 -- waflib/Tools/javaw.py | 5 -- waflib/extras/batched_cc.py | 13 ---- 9 files changed, 1 insertion(+), 169 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9a88f8db..14d005b8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,7 +1,7 @@ NEW IN WAF 1.8.0 ---------------- * Do not propagate external static libraries uselib vars #1320 -* +* Removed the Waf cache from the task level NEW IN WAF 1.7.11 ----------------- diff --git a/docs/book/execution.txt b/docs/book/execution.txt index 08b801fe..c3e15fec 100644 --- a/docs/book/execution.txt +++ b/docs/book/execution.txt @@ -370,7 +370,6 @@ Options: -j JOBS, --jobs=JOBS amount of parallel jobs (2) -k, --keep keep running happily even if errors are found -v, --verbose verbosity level -v -vv or -vvv [default: 0] - --nocache ignore the WAFCACHE (if set) --zones=ZONES debugging zones (task_gen, deps, tasks, etc) configure options: diff --git a/waflib/Build.py b/waflib/Build.py index 67ec02cb..424a7455 100644 --- a/waflib/Build.py +++ b/waflib/Build.py @@ -96,8 +96,6 @@ class BuildContext(Context.Context): self.jobs = Options.options.jobs self.targets = Options.options.targets self.keep = Options.options.keep - self.cache_global = Options.cache_global - self.nocache = Options.options.nocache self.progress_bar = Options.options.progress_bar ############ stuff below has not been reviewed diff --git a/waflib/Options.py b/waflib/Options.py index 4eb7f3dd..90fdf3cf 100644 --- a/waflib/Options.py +++ b/waflib/Options.py @@ -36,8 +36,6 @@ List of commands to execute extracted from the command-line. This list is consum """ lockfile = os.environ.get('WAFLOCK', '.lock-waf_%s_build' % sys.platform) -try: cache_global = os.path.abspath(os.environ['WAFCACHE']) -except KeyError: cache_global = '' platform = Utils.unversioned_sys_platform() @@ -56,7 +54,6 @@ class opt_parser(optparse.OptionParser): p('-j', '--jobs', dest='jobs', default=jobs, type='int', help='amount of parallel jobs (%r)' % jobs) p('-k', '--keep', dest='keep', default=0, action='count', help='keep running happily even if errors are found') p('-v', '--verbose', dest='verbose', default=0, action='count', help='verbosity level -v -vv or -vvv [default: 0]') - p('--nocache', dest='nocache', default=False, action='store_true', help='ignore the WAFCACHE (if set)') p('--zones', dest='zones', default='', action='store', help='debugging zones (task_gen, deps, tasks, etc)') gr = optparse.OptionGroup(self, 'configure options') diff --git a/waflib/Task.py b/waflib/Task.py index 99bbd54e..33ba3558 100644 --- a/waflib/Task.py +++ b/waflib/Task.py @@ -63,40 +63,6 @@ def f(tsk): return tsk.exec_command(lst, cwd=wd, env=env.env or None) ''' -def cache_outputs(cls): - """ - Task class decorator applied to all task classes by default unless they define the attribute 'nocache':: - - from waflib import Task - class foo(Task.Task): - nocache = True - - If bld.cache_global is defined and if the task instances produces output nodes, - the files will be copied into a folder in the cache directory - - The files may also be retrieved from that folder, if it exists - """ - m1 = cls.run - def run(self): - bld = self.generator.bld - if bld.cache_global and not bld.nocache: - if self.can_retrieve_cache(): - return 0 - return m1(self) - cls.run = run - - m2 = cls.post_run - def post_run(self): - bld = self.generator.bld - ret = m2(self) - if bld.cache_global and not bld.nocache: - self.put_files_cache() - return ret - cls.post_run = post_run - - return cls - - classes = {} "class tasks created by user scripts or Waf tools are kept in this dict name -> class object" @@ -128,9 +94,6 @@ class store_task_type(type): # getattr(cls, 'hcode') would look in the upper classes cls.hcode = Utils.h_fun(cls.run) - if not getattr(cls, 'nocache', None): - cls = cache_outputs(cls) - # be creative getattr(cls, 'register', classes)[name] = cls @@ -811,107 +774,6 @@ class Task(TaskBase): #print "task is not ready..." raise Errors.TaskNotReady('not ready') - def can_retrieve_cache(self): - """ - Used by :py:meth:`waflib.Task.cache_outputs` - - Retrieve build nodes from the cache - update the file timestamps to help cleaning the least used entries from the cache - additionally, set an attribute 'cached' to avoid re-creating the same cache files - - Suppose there are files in `cache/dir1/file1` and `cache/dir2/file2`: - - #. read the timestamp of dir1 - #. try to copy the files - #. look at the timestamp again, if it has changed, the data may have been corrupt (cache update by another process) - #. should an exception occur, ignore the data - """ - - if not getattr(self, 'outputs', None): - return None - - sig = self.signature() - ssig = Utils.to_hex(self.uid()) + Utils.to_hex(sig) - - # first try to access the cache folder for the task - dname = os.path.join(self.generator.bld.cache_global, ssig) - try: - t1 = os.stat(dname).st_mtime - except OSError: - return None - - for node in self.outputs: - orig = os.path.join(dname, node.name) - try: - shutil.copy2(orig, node.abspath()) - # mark the cache file as used recently (modified) - os.utime(orig, None) - except (OSError, IOError): - Logs.debug('task: failed retrieving file') - return None - - # is it the same folder? - try: - t2 = os.stat(dname).st_mtime - except OSError: - return None - - if t1 != t2: - return None - - for node in self.outputs: - node.sig = sig - if self.generator.bld.progress_bar < 1: - self.generator.bld.to_log('restoring from cache %r\n' % node.abspath()) - - self.cached = True - return True - - def put_files_cache(self): - """ - Used by :py:func:`waflib.Task.cache_outputs` to store the build files in the cache - """ - - # file caching, if possible - # try to avoid data corruption as much as possible - if getattr(self, 'cached', None): - return None - if not getattr(self, 'outputs', None): - return None - - sig = self.signature() - ssig = Utils.to_hex(self.uid()) + Utils.to_hex(sig) - dname = os.path.join(self.generator.bld.cache_global, ssig) - tmpdir = tempfile.mkdtemp(prefix=self.generator.bld.cache_global + os.sep + 'waf') - - try: - shutil.rmtree(dname) - except Exception: - pass - - try: - for node in self.outputs: - dest = os.path.join(tmpdir, node.name) - shutil.copy2(node.abspath(), dest) - except (OSError, IOError): - try: - shutil.rmtree(tmpdir) - except Exception: - pass - else: - try: - os.rename(tmpdir, dname) - except OSError: - try: - shutil.rmtree(tmpdir) - except Exception: - pass - else: - try: - os.chmod(dname, Utils.O755) - except Exception: - pass - def is_before(t1, t2): """ Return a non-zero value if task t1 is to be executed before task t2:: diff --git a/waflib/Tools/c_config.py b/waflib/Tools/c_config.py index 21417cbf..285aaf9e 100755 --- a/waflib/Tools/c_config.py +++ b/waflib/Tools/c_config.py @@ -1190,8 +1190,6 @@ def multicheck(self, *k, **kw): class par(object): def __init__(self): self.keep = False - self.cache_global = Options.cache_global - self.nocache = Options.options.nocache self.returned_tasks = [] self.task_sigs = {} def total(self): diff --git a/waflib/Tools/fc.py b/waflib/Tools/fc.py index 0eacf920..f81c03d5 100644 --- a/waflib/Tools/fc.py +++ b/waflib/Tools/fc.py @@ -160,10 +160,6 @@ class fcshlib(fcprogram): class fcprogram_test(fcprogram): """Custom link task to obtain the compiler outputs for fortran configuration tests""" - def can_retrieve_cache(self): - """This task is always executed""" - return False - def runnable_status(self): """This task is always executed""" ret = super(fcprogram_test, self).runnable_status() diff --git a/waflib/Tools/javaw.py b/waflib/Tools/javaw.py index 5e6c6d0d..2babe0d8 100644 --- a/waflib/Tools/javaw.py +++ b/waflib/Tools/javaw.py @@ -236,11 +236,6 @@ class javac(Task.Task): """ color = 'BLUE' - nocache = True - """ - The .class files cannot be put into a cache at the moment - """ - vars = ['CLASSPATH', 'JAVACFLAGS', 'JAVAC', 'OUTDIR'] """ The javac task will be executed again if the variables CLASSPATH, JAVACFLAGS, JAVAC or OUTDIR change. diff --git a/waflib/extras/batched_cc.py b/waflib/extras/batched_cc.py index 39ba6e33..e7bef64b 100644 --- a/waflib/extras/batched_cc.py +++ b/waflib/extras/batched_cc.py @@ -141,21 +141,8 @@ for x in ['c', 'cxx']: #self.executed=1 pass - def can_retrieve_cache(self): - if self.old_can_retrieve_cache(): - for m in self.generator.allmasters: - try: - m.slaves.remove(self) - except ValueError: - pass #this task wasn't included in that master - return 1 - else: - return None - setattr(t, 'oldrun', t.__dict__['run']) setattr(t, 'run', run) setattr(t, 'old_post_run', t.post_run) setattr(t, 'post_run', post_run) - setattr(t, 'old_can_retrieve_cache', t.can_retrieve_cache) - setattr(t, 'can_retrieve_cache', can_retrieve_cache)