waf/waflib/extras/win32_opts.py

#! /usr/bin/env python
# encoding: utf-8

"""
Windows-specific optimizations

This module can help reducing the overhead of listing files on windows (more than 10000 files).
"""

import os
try:
	import cPickle
except ImportError:
	import pickle as cPickle
from waflib import Utils, Build, Context, Node, Logs

try:
	TP = '%s\\*'.decode('ascii')
except AttributeError:
	TP = '%s\\*'

if Utils.is_win32:
	from waflib.extras import md5_tstamp
	import ctypes, ctypes.wintypes

	FindFirstFile        = ctypes.windll.kernel32.FindFirstFileW
	FindNextFile         = ctypes.windll.kernel32.FindNextFileW
	FindClose            = ctypes.windll.kernel32.FindClose
	FILE_ATTRIBUTE_DIRECTORY = 0x10
	INVALID_HANDLE_VALUE = -1
	UPPER_FOLDERS = ('.', '..')
	try:
		UPPER_FOLDERS = [unicode(x) for x in UPPER_FOLDERS]
	except NameError:
		pass

	def cached_hash_file(self):
		try:
			cache = self.ctx.cache_listdir_cache_hash_file
		except AttributeError:
			cache = self.ctx.cache_listdir_cache_hash_file = {}

		if id(self.parent) in cache:
			try:
				t = cache[id(self.parent)][self.name]
			except KeyError:
				raise IOError('Not a file')
		else:
			# an opportunity to list the files and the timestamps at once
			findData = ctypes.wintypes.WIN32_FIND_DATAW()
			find     = FindFirstFile(TP % self.parent.abspath(), ctypes.byref(findData))

			if find == INVALID_HANDLE_VALUE:
				cache[id(self.parent)] = {}
				raise IOError('Not a file')

			cache[id(self.parent)] = lst_files = {}
			try:
				while True:
					if findData.cFileName not in UPPER_FOLDERS:
						thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY
						if not thatsadir:
							ts = findData.ftLastWriteTime
							d = (ts.dwLowDateTime << 32) | ts.dwHighDateTime
							lst_files[str(findData.cFileName)] = d
					if not FindNextFile(find, ctypes.byref(findData)):
						break
			except Exception as e:
				cache[id(self.parent)] = {}
				raise IOError('Not a file')
			finally:
				FindClose(find)
			t = lst_files[self.name]

		fname = self.abspath()
		if fname in Build.hashes_md5_tstamp:
			if Build.hashes_md5_tstamp[fname][0] == t:
				return Build.hashes_md5_tstamp[fname][1]

		try:
			fd = os.open(fname, os.O_BINARY | os.O_RDONLY | os.O_NOINHERIT)
		except OSError:
			raise IOError('Cannot read from %r' % fname)
		f = os.fdopen(fd, 'rb')
		m = Utils.md5()
		rb = 1
		try:
			while rb:
				rb = f.read(200000)
				m.update(rb)
		finally:
			f.close()

		# ensure that the cache is overwritten
		Build.hashes_md5_tstamp[fname] = (t, m.digest())
		return m.digest()
	Node.Node.cached_hash_file = cached_hash_file

	def get_bld_sig_win32(self):
		try:
			return self.ctx.hash_cache[id(self)]
		except KeyError:
			pass
		except AttributeError:
			self.ctx.hash_cache = {}

		if not self.is_bld():
			if self.is_child_of(self.ctx.srcnode):
				self.sig = self.cached_hash_file()
			else:
				self.sig = Utils.h_file(self.abspath())
		self.ctx.hash_cache[id(self)] = ret = self.sig
		return ret
	Node.Node.get_bld_sig = get_bld_sig_win32

	def isfile_cached(self):
		# optimize for nt.stat calls, assuming there are many files for few folders
		try:
			cache = self.__class__.cache_isfile_cache
		except AttributeError:
			cache = self.__class__.cache_isfile_cache = {}

		try:
			c1 = cache[id(self.parent)]
		except KeyError:
			c1 = cache[id(self.parent)] = []

			curpath = self.parent.abspath()
			findData = ctypes.wintypes.WIN32_FIND_DATAW()
			find     = FindFirstFile(TP % curpath, ctypes.byref(findData))

			if find == INVALID_HANDLE_VALUE:
				Logs.error("invalid win32 handle isfile_cached %r" % self.abspath())
				return os.path.isfile(self.abspath())

			try:
				while True:
					if findData.cFileName not in UPPER_FOLDERS:
						thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY
						if not thatsadir:
							c1.append(str(findData.cFileName))
					if not FindNextFile(find, ctypes.byref(findData)):
						break
			except Exception as e:
				Logs.error('exception while listing a folder %r %r' % (self.abspath(), e))
				return os.path.isfile(self.abspath())
			finally:
				FindClose(find)
		return self.name in c1
	Node.Node.isfile_cached = isfile_cached

	def find_or_declare_win32(self, lst):
		# assuming that "find_or_declare" is called before the build starts, remove the calls to os.path.isfile
		if isinstance(lst, str):
			lst = [x for x in Node.split_path(lst) if x and x != '.']

		node = self.get_bld().search(lst)
		if node:
			if not node.isfile_cached():
				node.sig = None
				try:
					node.parent.mkdir()
				except OSError:
					pass
			return node
		self = self.get_src()
		node = self.find_node(lst)
		if node:
			if not node.isfile_cached():
				node.sig = None
				try:
					node.parent.mkdir()
				except OSError:
					pass
			return node
		node = self.get_bld().make_node(lst)
		node.parent.mkdir()
		return node
	Node.Node.find_or_declare = find_or_declare_win32