add distnet extras tool, and assorted examples (WIP)

vs. builds on shared folders...

playground/distnet/README.rst

@@ -0,0 +1,43 @@
+#######
+distnet
+#######
+
+This example provides an example of the `remote` extras tool,
+used to build and share binary packages in the context of an intranet.
+
+Usage
+#####
+
+Run the following in order in three distinct consoles:
+
+1. start the server::
+
+     cd server && ./start.sh
+
+2. publish a package::
+   
+     cd app && waf configure_all build_all package publish
+
+3. use a package in a project::
+   
+     cd app2 && waf configure_all build_all
+
+Features
+########
+
+- a simple cgi server helps uploading/distributing the files
+- headers can be redistributed
+- binary data can be redistributed
+- configuration scripts can be redistributed along with build rules
+- packages are compressed on the server
+
+Limitations
+###########
+
+- Waf and Python cannot be distributed as a packages (may require another process or an auto-update system)
+- all dependencies must be specified at the moment, and there is no consistency verification
+- once a folder is written to the cache it is never updated again
+- there is no integrity verification aside from the package compresssion
+- files submitted must not be small enough
+- no server security (do it yourself!)
+

playground/distnet/app/head.h

@@ -0,0 +1,5 @@
+#ifndef pouet
+	#error "project not configured properly"
+#endif
+
+int foo();

playground/distnet/app/main.c

@@ -0,0 +1,4 @@
+
+int foo() {
+	return 1095672;
+}

playground/distnet/app/requires.txt

@@ -0,0 +1 @@
+# nothing yet

playground/distnet/app/waf_pouet.py

@@ -0,0 +1,16 @@
+# module exported and used for configuring the package pouet
+
+import os
+
+def options(opt):
+	# project-specific options go here
+	pass
+
+def configure(conf):
+	conf.env.append_value('DEFINES_pouet', 'pouet=1')
+	conf.env.append_value('INCLUDES_pouet', os.path.dirname(os.path.abspath(__file__)))
+
+def build(bld):
+	# project-specific build targets go here
+	pass
+

playground/distnet/app/wscript

@@ -0,0 +1,44 @@
+#! /usr/bin/env python
+# encoding: utf-8
+
+VERSION='1.0.0'
+APPNAME='app'
+
+top = '.'
+out = 'build'
+
+from waflib.extras import remote # optional
+from waflib.extras import distnet
+
+variants = [
+'linux_64_debug',
+'linux_64_release',
+'linux_32_debug',
+'linux_32_release',
+]
+
+def options(opt):
+	opt.load('distnet')
+	opt.load('compiler_c')
+
+def configure(conf):
+	conf.load('distnet')
+	conf.load('compiler_c')
+
+def build(bld):
+	bld.shlib(source='main.c', target='pouet', includes='.')
+
+def package(ctx):
+	for v in variants:
+		tar = 'build/%s.tarfile' % v
+		inputs = ['build/%s/libpouet.so' % v]
+		ctx.make_tarfile(tar, inputs)
+	ctx.make_tarfile('build/noarch.tarfile', ['head.h', 'waf_pouet.py'])
+
+def test_download(ctx):
+	import urllib
+	data = urllib.urlencode([('pkgname', APPNAME), ('pkgver', VERSION), ('pkgfile', 'noarch.tarfile')])
+	def hook(a, b, c):
+		pass
+	urllib.urlretrieve('http://localhost:8000/cgi-bin/download.py', 'x', hook, data)
+

playground/distnet/app2/main.c

@@ -0,0 +1,6 @@
+#include "head.h"
+
+int main()
+{
+	return 0;
+}

playground/distnet/app2/requires.txt

@@ -0,0 +1 @@
+app,1.0.*,a=32

playground/distnet/app2/wscript

@@ -0,0 +1,37 @@
+#! /usr/bin/env python
+# encoding: utf-8
+
+VERSION='0.0.1'
+APPNAME='app2'
+
+top = '.'
+out = 'build'
+
+from waflib.extras import remote # optional
+from waflib.extras import distnet
+
+variants = [
+	'linux_64_debug',
+	'linux_64_release',
+	'linux_32_debug',
+	'linux_32_release',
+]
+
+def options(opt):
+	opt.load('distnet')
+	opt.load('compiler_c')
+
+def configure(conf):
+	conf.load('distnet')
+	conf.load('compiler_c')
+
+def build(bld):
+	bld.program(source='main.c', target='app2', includes='.', use='pouet')
+
+def package(ctx):
+	for v in variants:
+		tar = 'build/%s.tarfile' % v
+		inputs = ['build/%s/libpouet.so' % v]
+		ctx.make_tarfile(tar, inputs)
+	ctx.make_tarfile('build/noarch.tarfile', ['head.h'])
+

playground/distnet/server/cgi-bin/download.py

@@ -0,0 +1,41 @@
+#! /usr/bin/env python
+
+import os, sys
+import cgi, cgitb
+cgitb.enable()
+
+PKGDIR = os.environ.get('PKGDIR', os.path.abspath('../packages'))
+
+form = cgi.FieldStorage()
+def getvalue(x):
+	v = form.getvalue(x)
+	if not v:
+		print("Status: 413\ncontent-type: text/plain\n\nmissing %s\n" % x)
+	return v
+
+pkgname = getvalue('pkgname')
+pkgver = getvalue('pkgver')
+pkgfile = getvalue('pkgfile')
+
+filename = os.path.join(PKGDIR, pkgname, pkgver, pkgfile)
+if not os.path.exists(filename):
+	filename = filename + '.tarfile'
+
+if not os.path.exists(filename):
+	print("Status: 404\ncontent-type: text/plain\n\nInvalid package %r\n" % filename)
+
+length = os.stat(filename).st_size
+
+print "Content-Type: application/octet-stream"
+print "Content-Disposition: attachment; filename=f.bin"
+print "Content-length: %s" % length
+print ""
+
+with open(filename, 'rb') as f:
+	while True:
+		buf = f.read(8192)
+		if buf:
+			sys.stdout.write(buf)
+		else:
+			break
+

playground/distnet/server/cgi-bin/resolve.py

@@ -0,0 +1,25 @@
+#! /usr/bin/env python
+
+import os, sys
+import cgi, cgitb
+cgitb.enable()
+
+PKGDIR = os.environ.get('PKGDIR', os.path.abspath('../packages'))
+if not 'DISTNETCACHE' in os.environ:
+	os.environ['DISTNETCACHE'] = PKGDIR
+
+d = os.path.dirname
+base = d(d(d(d(d(os.path.abspath(__file__))))))
+sys.path.append(base)
+
+from waflib.extras import distnet
+
+form = cgi.FieldStorage()
+
+text = form.getvalue('text')
+distnet.packages.local_resolve(text)
+
+print '''Content-Type: text/plain
+
+%s''' % distnet.packages.get_results()
+

playground/distnet/server/cgi-bin/upload.py

@@ -0,0 +1,54 @@
+#! /usr/bin/env python
+
+import os, sys, tempfile, shutil, hashlib, tarfile
+import cgi, cgitb
+cgitb.enable()
+
+PKGDIR = os.environ.get('PKGDIR', os.path.abspath('../packages'))
+
+# Upload a package to the package directory.
+# It is meant to contain a list of tar packages:
+#
+# PKGDIR/pkgname/pkgver/common.tar
+# PKGDIR/pkgname/pkgver/arch1.tar
+# PKGDIR/pkgname/pkgver/arch2.tar
+# ...
+
+form = cgi.FieldStorage()
+def getvalue(x):
+	v = form.getvalue(x)
+	if not v:
+		print("Status: 413\ncontent-type: text/plain\n\nmissing %s\n" % x)
+	return v
+
+pkgname = getvalue('pkgname')
+pkgver = getvalue('pkgver')
+pkgdata = getvalue('pkgdata')
+# pkghash = getvalue('pkghash') # TODO provide away to verify file hashes and signatures?
+
+up = os.path.join(PKGDIR, pkgname)
+dest = os.path.join(up, pkgver)
+if os.path.exists(dest):
+	print("Status: 409\ncontent-type: text/plain\n\nPackage %r already exists!\n" % dest)
+else:
+	if not os.path.isdir(up):
+		os.makedirs(up)
+
+	tmp = tempfile.mkdtemp(dir=up)
+	try:
+		tf = os.path.join(tmp, 'some_temporary_file')
+		with open(tf, 'wb') as f:
+			f.write(pkgdata)
+		with tarfile.open(tf) as f:
+			f.extractall(tmp)
+		os.remove(tf)
+		os.rename(tmp, dest)
+	finally:
+		# cleanup
+		try:
+			shutil.rmtree(tmp)
+		except Exception:
+			pass
+
+	print('''Content-Type: text/plain\n\nok''')
+

playground/distnet/server/start.sh

@@ -0,0 +1,4 @@
+#! /bin/sh
+
+python -m CGIHTTPServer
+

waflib/extras/distnet.py

@@ -0,0 +1,268 @@
+#! /usr/bin/env python
+# encoding: utf-8
+
+"""
+waf-powered distributed network builds, with a network cache.
+
+Caching files from a server has advantages over a NFS/Samba shared folder:
+
+- builds are much faster because they use local files
+- builds just continue to work in case of a network glitch
+- permissions are much simpler to manage
+
+TODO: python3 compatibility
+
+"""
+
+import os, urllib, urllib2, tarfile, collections, re, shutil, tempfile
+from waflib import Context, Configure, Utils, Logs
+
+DISTNETCACHE = os.environ.get('DISTNETCACHE', '/tmp/distnetcache')
+DISTNETSERVER = os.environ.get('DISTNETSERVER', 'http://localhost:8000/cgi-bin/')
+TARFORMAT = 'w:bz2'
+TIMEOUT=60
+
+re_com = re.compile('\s*#.*', re.M)
+
+def get_distnet_cache():
+	return getattr(Context.g_module, 'DISTNETCACHE', DISTNETCACHE)
+
+def get_server_url():
+	return getattr(Context.g_module, 'DISTNETSERVER', DISTNETSERVER)
+
+def get_download_url():
+	return '%s/download.py' % get_server_url()
+
+def get_upload_url():
+	return '%s/upload.py' % get_server_url()
+
+def get_resolve_url():
+	return '%s/resolve.py' % get_server_url()
+
+def send_package_name():
+	out = getattr(Context.g_module, 'out', 'build')
+	pkgfile = '%s/package_to_upload.tarfile' % out
+	return pkgfile
+
+class package(Context.Context):
+	fun = 'package'
+	cmd = 'package'
+
+	def execute(self):
+		try:
+			files = self.files
+		except AttributeError:
+			files = self.files = []
+
+		Context.Context.execute(self)
+		pkgfile = send_package_name()
+		if not pkgfile in self.files:
+			if not 'requires.txt' in self.files:
+				self.files.append('requires.txt')
+			self.make_tarfile(pkgfile, self.files, add_to_package=False)
+
+	def make_tarfile(self, filename, files, **kw):
+		if kw.get('add_to_package', True):
+			self.files.append(filename)
+
+		with tarfile.open(filename, TARFORMAT) as tar:
+			endname = os.path.split(filename)[-1]
+			endname = endname.split('.')[0] + '/'
+			for x in files:
+				tarinfo = tar.gettarinfo(x, x)
+				tarinfo.uid   = tarinfo.gid   = 0
+				tarinfo.uname = tarinfo.gname = 'root'
+				tarinfo.size = os.stat(x).st_size
+
+				# TODO - more archive creation options?
+				if kw.get('bare', True):
+					tarinfo.name = os.path.split(x)[1]
+				else:
+					tarinfo.name = endname + x # todo, if tuple, then..
+				Logs.debug("adding %r to %s" % (tarinfo.name, filename))
+				with open(x, 'rb') as f:
+					tar.addfile(tarinfo, f)
+		Logs.info('Created %s' % filename)
+
+class publish(Context.Context):
+	fun = 'publish'
+	cmd = 'publish'
+	def execute(self):
+		if hasattr(Context.g_module, 'publish'):
+			Context.Context.execute(self)
+		mod = Context.g_module
+
+		rfile = getattr(self, 'rfile', send_package_name())
+		if not os.path.isfile(rfile):
+			self.fatal('Create the release file with "waf release" first! %r' % rfile)
+
+		fdata = Utils.readf(rfile, m='rb')
+		data = urllib.urlencode([('pkgdata', fdata), ('pkgname', mod.APPNAME), ('pkgver', mod.VERSION)])
+
+		req = urllib2.Request(get_upload_url(), data)
+		response = urllib2.urlopen(req, timeout=TIMEOUT)
+		data = response.read().strip()
+
+		if data != 'ok':
+			self.fatal('Could not publish the package %r' % data)
+
+
+class pkg(object):
+	pass
+	# name              foo
+	# version           1.0.0
+	# required_version  1.0.*
+	# localfolder       /tmp/packages/foo/1.0/
+
+class package_reader(object):
+	def read_packages(self, filename='requires.txt'):
+		txt = Utils.readf(filename).strip()
+		self.compute_dependencies(filename)
+
+	def read_package_string(self, txt):
+		if txt is None:
+			Logs.error('Hahaha, None!')
+		self.pkg_list = []
+		txt = re.sub(re_com, '', txt)
+		lines = txt.splitlines()
+		for line in lines:
+			if not line:
+				continue
+			p = pkg()
+			p.required_line = line
+			lst = line.split(',')
+			p.name = lst[0]
+			p.requested_version = lst[1]
+			self.pkg_list.append(p)
+			for k in lst:
+				a, b, c = k.partition('=')
+				if a and c:
+					setattr(p, a, c)
+
+	def compute_dependencies(self, filename='requires.txt'):
+		text = Utils.readf(filename)
+		data = urllib.urlencode([('text', text)])
+		req = urllib2.Request(get_resolve_url(), data)
+		try:
+			response = urllib2.urlopen(req, timeout=TIMEOUT)
+		except urllib2.URLError as e:
+			Logs.warn('The package server is down! %r' % e)
+			self.local_resolve(text)
+		else:
+			ret = response.read()
+			print ret
+			self.read_package_string(ret)
+
+		errors = False
+		for p in self.pkg_list:
+			if getattr(p, 'error', ''):
+				Logs.error(p.error)
+				errors = True
+		if errors:
+			raise ValueError('Requirements could not be satisfied!')
+
+	def get_results(self):
+		buf = []
+		for x in self.pkg_list:
+			buf.append('%s,%s' % (x.name, x.requested_version))
+			for y in ('error', 'version'):
+				if hasattr(x, y):
+					buf.append(',%s=%s' % (y, getattr(x, y)))
+			buf.append('\n')
+		return ''.join(buf)
+
+	def local_resolve(self, text):
+		self.read_package_string(text)
+		for p in self.pkg_list:
+
+			pkgdir = os.path.join(get_distnet_cache(), p.name)
+			try:
+				versions = os.listdir(pkgdir)
+			except OSError:
+				p.error = 'Directory %r does not exist' % pkgdir
+				continue
+
+			vname = p.requested_version.replace('*', '.*')
+			rev = re.compile(vname, re.M)
+			versions = [x for x in versions if rev.match(x)]
+			versions.sort()
+
+			try:
+				p.version = versions[0]
+			except IndexError:
+				p.error = 'There is no package that satisfies %r %r' % (p.name, p.requested_version)
+
+	def download_to_file(self, p, subdir, tmp):
+		data = urllib.urlencode([('pkgname', p.name), ('pkgver', p.version), ('pkgfile', subdir)])
+		req = urllib2.urlopen(get_download_url(), data, timeout=TIMEOUT)
+		with open(tmp, 'wb') as f:
+			while True:
+				buf = req.read(8192)
+				if not buf:
+					break
+				f.write(buf)
+
+	def extract_tar(self, subdir, pkgdir, tmpfile):
+		with tarfile.open(tmpfile) as f:
+			temp = tempfile.mkdtemp(dir=pkgdir)
+			try:
+				f.extractall(temp)
+				os.rename(temp, os.path.join(pkgdir, subdir))
+			finally:
+				try:
+					shutil.rmtree(temp)
+				except Exception:
+					pass
+
+	def get_pkg_dir(self, pkg, subdir):
+		pkgdir = os.path.join(get_distnet_cache(), pkg.name, pkg.version)
+		if not os.path.isdir(pkgdir):
+			os.makedirs(pkgdir)
+
+		target = os.path.join(pkgdir, subdir)
+		if os.path.exists(target):
+			return target
+
+		(fd, tmp) = tempfile.mkstemp(dir=pkgdir)
+		try:
+			os.close(fd)
+			self.download_to_file(pkg, subdir, tmp)
+			if subdir == 'requires.txt':
+				os.rename(tmp, target)
+			else:
+				self.extract_tar(subdir, pkgdir, tmp)
+		finally:
+			try:
+				os.remove(tmp)
+			except OSError as e:
+				pass
+
+		return target
+
+	def __iter__(self):
+		if not hasattr(self, 'pkg_list'):
+			self.read_packages()
+			self.compute_dependencies()
+		for x in self.pkg_list:
+			yield x
+		raise StopIteration
+
+packages = package_reader()
+
+def load_tools(ctx, extra):
+	global packages
+	for pkg in packages:
+		packages.get_pkg_dir(pkg, extra)
+		noarchdir = packages.get_pkg_dir(pkg, 'noarch')
+		#sys.path.append(noarchdir)
+		for x in os.listdir(noarchdir):
+			if x.startswith('waf_') and x.endswith('.py'):
+				ctx.load(x.rstrip('.py'), tooldir=noarchdir)
+
+def options(opt):
+	packages.read_packages()
+	load_tools(opt, 'requires.txt')
+
+def configure(conf):
+	load_tools(conf, conf.variant)
+