From d8b2bbc3635dc9cc640beecc74a990ce7c15dfc3 Mon Sep 17 00:00:00 2001 From: Thomas Nagy Date: Mon, 2 Apr 2012 01:11:21 +0200 Subject: [PATCH] win32 optimizations --- waflib/extras/win32_opts.py | 192 ++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 waflib/extras/win32_opts.py diff --git a/waflib/extras/win32_opts.py b/waflib/extras/win32_opts.py new file mode 100644 index 00000000..3ffc2d88 --- /dev/null +++ b/waflib/extras/win32_opts.py @@ -0,0 +1,192 @@ +#! /usr/bin/env python +# encoding: utf-8 + +""" +Windows-specific optimizations +""" + +import os +try: import cPickle +except: import pickle as cPickle +from waflib import Utils, Build, Context, Node, Logs + +if Utils.is_win32: + import ctypes, ctypes.wintypes + + Context.DBFILE += '_md5tstamp' + + try: + Build.BuildContext.store_real + except AttributeError: + pass + else: + raise ValueError('win32opts has been loaded twice (or the incompatible md5_tstamp tool was loaded before)') + + Build.hashes_md5_tstamp = {} + Build.SAVED_ATTRS.append('hashes_md5_tstamp') + def store(self): + # save the hash cache as part of the default pickle file + self.hashes_md5_tstamp = Build.hashes_md5_tstamp + self.store_real() + Build.BuildContext.store_real = Build.BuildContext.store + Build.BuildContext.store = store + + def restore(self): + # we need a module variable for h_file below + self.restore_real() + try: + Build.hashes_md5_tstamp = self.hashes_md5_tstamp or {} + except Exception as e: + Build.hashes_md5_tstamp = {} + + Build.BuildContext.restore_real = Build.BuildContext.restore + Build.BuildContext.restore = restore + + FindFirstFile = ctypes.windll.kernel32.FindFirstFileW + FindNextFile = ctypes.windll.kernel32.FindNextFileW + FindClose = ctypes.windll.kernel32.FindClose + FILE_ATTRIBUTE_DIRECTORY = 0x10 + INVALID_HANDLE_VALUE = -1 + UPPER_FOLDERS = ('.', '..') + try: + UPPER_FOLDERS = [unicode(x) for x in UPPER_FOLDERS] + except NameError: + pass + + def cached_hash_file(self): + try: + cache = self.ctx.cache_listdir_cache_hash_file + except AttributeError: + cache = self.ctx.cache_listdir_cache_hash_file = {} + + if id(self.parent) in cache: + try: + t = cache[id(self.parent)][self.name] + except KeyError: + raise IOError('Not a file') + else: + # an opportunity to list the files and the timestamps at once + findData = ctypes.wintypes.WIN32_FIND_DATAW() + find = FindFirstFile(u'%s\\*' % self.parent.abspath(), ctypes.byref(findData)) + + if find == INVALID_HANDLE_VALUE: + cache[id(self.parent)] = {} + raise IOError('Not a file') + + cache[id(self.parent)] = lst_files = {} + try: + while True: + if findData.cFileName not in UPPER_FOLDERS: + thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY + if not thatsadir: + ts = findData.ftLastWriteTime + d = (ts.dwLowDateTime << 32) | ts.dwHighDateTime + lst_files[str(findData.cFileName)] = d + if not FindNextFile(find, ctypes.byref(findData)): + break + except Exception, e: + cache[id(self.parent)] = {} + raise IOError('Not a file') + finally: + FindClose(find) + t = lst_files[self.name] + + filename = self.abspath() + if filename in Build.hashes_md5_tstamp: + if Build.hashes_md5_tstamp[filename][0] == t: + return Build.hashes_md5_tstamp[filename][1] + m = Utils.md5() + + with open(filename, 'rb') as f: + read = 1 + while read: + read = f.read(100000) + m.update(read) + + # ensure that the cache is overwritten + Build.hashes_md5_tstamp[filename] = (t, m.digest()) + return m.digest() + Node.Node.cached_hash_file = cached_hash_file + + def get_bld_sig_win32(self): + try: + return self.ctx.hash_cache[id(self)] + except KeyError: + pass + except AttributeError: + self.ctx.hash_cache = {} + + if not self.is_bld(): + if self.is_child_of(self.ctx.srcnode): + self.sig = self.cached_hash_file() + else: + self.sig = Utils.h_file(self.abspath()) + self.ctx.hash_cache[id(self)] = ret = self.sig + return ret + Node.Node.get_bld_sig = get_bld_sig_win32 + + def isfile_cached(self): + # optimize for nt.stat calls, assuming there are many files for few folders + try: + cache = self.__class__.cache_isfile_cache + except AttributeError: + cache = self.__class__.cache_isfile_cache = {} + + try: + c1 = cache[id(self.parent)] + except KeyError: + c1 = cache[id(self.parent)] = [] + + curpath = self.parent.abspath() + findData = ctypes.wintypes.WIN32_FIND_DATAW() + find = FindFirstFile(u'%s\\*' % curpath, ctypes.byref(findData)) + + if find == INVALID_HANDLE_VALUE: + Logs.error("invalid win32 handle isfile_cached %r" % self.abspath()) + return os.path.isfile(self.abspath()) + + try: + while True: + if findData.cFileName not in UPPER_FOLDERS: + thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY + if not thatsadir: + c1.append(str(findData.cFileName)) + if not FindNextFile(find, ctypes.byref(findData)): + break + except Exception, e: + Logs.error('exception while listing a folder %r %r' % (self.abspath(), e)) + return os.path.isfile(self.abspath()) + finally: + FindClose(find) + return self.name in c1 + Node.Node.isfile_cached = isfile_cached + + def find_or_declare_win32(self, lst): + # assuming that "find_or_declare" is called before the build starts, remove the calls to os.path.isfile + if isinstance(lst, str): + lst = [x for x in Node.split_path(lst) if x and x != '.'] + + node = self.get_bld().search(lst) + if node: + if not node.isfile_cached(): + node.sig = None + try: + node.parent.mkdir() + except: + pass + return node + self = self.get_src() + node = self.find_node(lst) + if node: + if not node.isfile_cached(): + node.sig = None + try: + node.parent.mkdir() + except: + pass + return node + node = self.get_bld().make_node(lst) + node.parent.mkdir() + return node + Node.Node.find_or_declare = find_or_declare_win32 +