waf-zip: zip-packed waf

Ability to use a waflib.zip file with waf-zip entry point

Changes:

- created waf-zip entry point
- waflib.zip can created by doing::

     zip waflib -9 waflib/*.py waflib/Tools/*.py waflib/extras/*.py

- the wscript also builds waflib.zip
- allow selecting which core tools to use (saves 31k for C/C++ only)
  By default, everything is included.
- altered module loading functions to allow loading modules from a zip file.

waflib/Context.py

@@ -528,10 +528,28 @@ class Context(ctx):
 
 	def load_special_tools(self, var, ban=[]):
 		global waf_dir
-		lst = self.root.find_node(waf_dir).find_node('waflib/extras').ant_glob(var)
-		for x in lst:
-			if not x.name in ban:
-				load_tool(x.name.replace('.py', ''))
+		if os.path.isdir(waf_dir):
+			lst = self.root.find_node(waf_dir).find_node('waflib/extras').ant_glob(var)
+			for x in lst:
+				if not x.name in ban:
+					load_tool(x.name.replace('.py', ''))
+		else:
+			from zipfile import PyZipFile
+			import re
+			waflibs = PyZipFile(waf_dir)
+			lst = waflibs.namelist()
+			for x in lst:
+				if not re.match("waflib/extras/%s" % var.replace("*", ".*"), var):
+					continue
+				f = os.path.basename(x)
+				doban = False
+				for b in ban:
+					r = b.replace("*", ".*")
+					if re.match(b, f):
+						doban = True
+				if not doban:
+					f = f.replace('.py', '')
+					load_tool(f)
 
 cache_modules = {}
 """
@@ -595,21 +613,17 @@ def load_tool(tool, tooldir=None):
 			for d in tooldir:
 				sys.path.remove(d)
 	else:
-		global waf_dir
-		try:
-			os.stat(os.path.join(waf_dir, 'waflib', 'extras', tool + '.py'))
-		except OSError:
+		for x in ('waflib.extras.%s', 'waflib.Tools.%s', 'waflib.%s', '%s'):
 			try:
-				os.stat(os.path.join(waf_dir, 'waflib', 'Tools', tool + '.py'))
-			except OSError:
-				d = tool # user has messed with sys.path
-			else:
-				d = 'waflib.Tools.%s' % tool
-		else:
-			d = 'waflib.extras.%s' % tool
-
-		__import__(d)
-		ret = sys.modules[d]
+				__import__(x % tool)
+				break
+			except ImportError as e:
+				if not e.args[0].endswith(tool):
+					raise
+				x = None
+		if x is None: # raise an exception
+			__import__(tool)
+		ret = sys.modules[x % tool]
 		Context.tools[tool] = ret
 		return ret
 

wscript

@@ -111,6 +111,8 @@ def options(opt):
 		dest='strip_comments')
 	opt.add_option('--tools', action='store', help='Comma-separated 3rd party tools to add, eg: "compat,ocaml" [Default: "compat15"]',
 		dest='add3rdparty', default='compat15')
+	opt.add_option('--coretools', action='store', help='Comma-separated core tools to add, eg: "vala,tex" [Default: all of them]',
+		dest='coretools', default='default')
 	opt.add_option('--prelude', action='store', help='Code to execute before calling waf', dest='prelude', default=PRELUDE)
 	opt.load('python')
 
@@ -249,14 +251,12 @@ def create_waf(*k, **kw):
 	mw = 'tmp-waf-'+VERSION
 	print("-> preparing %r" % mw)
 
-	import tarfile, re
+	import tarfile, re, zipfile
 
 	zipType = Options.options.zip.strip().lower()
 	if zipType not in zip_types:
 		zipType = zip_types[0]
 
-	#open a file as tar.[extension] for writing
-	tar = tarfile.open('%s.tar.%s' % (mw, zipType), "w:%s" % zipType)
 
 	files = []
 	add3rdparty = []
@@ -266,18 +266,28 @@ def create_waf(*k, **kw):
 		else:
 			add3rdparty.append(x + '.py')
 
+	coretools = []
+	for x in Options.options.coretools.split(','):
+		coretools.append(x + '.py')
+
 	for d in '. Tools extras'.split():
 		dd = os.path.join('waflib', d)
 		for k in os.listdir(dd):
 			if k == '__init__.py':
 				files.append(os.path.join(dd, k))
 				continue
+			if d == 'Tools' and Options.options.coretools != 'default':
+				if not k in coretools:
+					continue
 			if d == 'extras':
 				if not k in add3rdparty:
 					continue
 			if k.endswith('.py'):
 				files.append(os.path.join(dd, k))
 
+	#open a file as tar.[extension] for writing
+	tar = tarfile.open('%s.tar.%s' % (mw, zipType), "w:%s" % zipType)
+	z = zipfile.ZipFile("zip/waflib.zip", "w", compression=zipfile.ZIP_DEFLATED)
 	for x in files:
 		tarinfo = tar.gettarinfo(x, x)
 		tarinfo.uid   = tarinfo.gid   = 0
@@ -288,8 +298,18 @@ def create_waf(*k, **kw):
 		if os.path.isabs(x):
 			tarinfo.name = 'waflib/extras/' + os.path.split(x)[1]
 
+		print("   adding %s as %s" % (x, tarinfo.name))
+		
+		def dest(x):
+			if os.path.isabs(x):
+				return os.path.join("extras", os.path.basename(x))
+			else:
+				return os.path.normpath(os.path.relpath(x, "."))
+
+		z.write(x, dest(x))
 		tar.addfile(tarinfo, code)
 	tar.close()
+	z.close()
 
 	f = open('waf-light', 'rU')
 	try:

zip/.gitignore

@@ -0,0 +1 @@
+waflib.zip

zip/waf-zip

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# encoding: ISO8859-1
+# Thomas Nagy, 2005-2011
+
+"""
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. The name of the author may not be used to endorse or promote products
+   derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR "AS IS" AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+"""
+
+import os, sys
+
+if sys.hexversion<0x206000f:
+	raise ImportError('Python >= 2.6 is required to run waf-zip')
+
+wafzip = os.path.join(os.path.dirname(sys.argv[0]), "waflib.zip")
+sys.path.insert(0, wafzip)
+
+from waflib import Context
+
+if __name__ == '__main__':
+	from waflib import Scripting
+	Scripting.waf_entry_point(os.getcwd(), Context.WAFVERSION, wafzip)
+