diff --git a/playground/distnet/app/wscript b/playground/distnet/app/wscript index 54137012..b4e10957 100644 --- a/playground/distnet/app/wscript +++ b/playground/distnet/app/wscript @@ -35,10 +35,39 @@ def package(ctx): ctx.make_tarfile(tar, inputs) ctx.make_tarfile('build/noarch.tarfile', ['head.h', 'waf_pouet.py']) -def test_download(ctx): - import urllib - data = urllib.urlencode([('pkgname', APPNAME), ('pkgver', VERSION), ('pkgfile', 'noarch.tarfile')]) - def hook(a, b, c): - pass - urllib.urlretrieve('http://localhost:8000/cgi-bin/download.py', 'x', hook, data) +# testing.......................................................................................... +# + +from waflib import Utils +class problem(object): + def __init__(self): + self.reader = distnet.package_reader() + #self.reader.debug = 1 + self.vtable = {self.reader.myproject : [self.reader.myversion]} + + def add_constraint(self, pkgname, pkgver, text): + self.reader.cache_constraints[(pkgname, pkgver)] = distnet.parse_constraints(text) + + def set_versions(self, pkgname, versions): + self.vtable[pkgname] = Utils.to_list(versions) + +def test(ctx): + p = problem() + p.add_constraint(APPNAME, VERSION, 'app1,1.0.*\napp2,1.0.*') + p.add_constraint('app1', '1.0.0', 'app3,1.0.*,') + p.add_constraint('app2', '1.0.0', 'app3,2.0.*,') + p.add_constraint('app3', '1.0.0', '') + p.add_constraint('app3', '2.0.0', '') + + p.set_versions('app1', '1.0.0') + p.set_versions('app2', '1.0.0') + p.set_versions('app3', '1.0.0 2.0.0') + + versions, constraints = p.reader.solve(p.vtable, {}, p.reader.myproject, p.reader.myversion) + #print versions + #print constraints + #for a, b in constraints.items(): + # print a, b + p.reader.constraints = p.reader.solution_to_constraints(versions, constraints) + p.reader.check_errors() diff --git a/playground/distnet/server/cgi-bin/resolve.py b/playground/distnet/server/cgi-bin/resolve.py index 6cdf717d..23e48011 100755 --- a/playground/distnet/server/cgi-bin/resolve.py +++ b/playground/distnet/server/cgi-bin/resolve.py @@ -17,9 +17,12 @@ from waflib.extras import distnet form = cgi.FieldStorage() text = form.getvalue('text') -distnet.packages.local_resolve(text) +distnet.packages.debug = 0 +distnet.packages.constraints = distnet.packages.local_resolve(text) +results = distnet.packages.get_results() -print '''Content-Type: text/plain - -%s''' % distnet.packages.get_results() +print "Content-Type: text/plain" +print "" +print "" +print results diff --git a/waflib/extras/distnet.py b/waflib/extras/distnet.py index f44580f9..add765a4 100644 --- a/waflib/extras/distnet.py +++ b/waflib/extras/distnet.py @@ -12,6 +12,7 @@ Caching files from a server has advantages over a NFS/Samba shared folder: """ import os, urllib, tarfile, re, shutil, tempfile, sys +from collections import OrderedDict from waflib import Context, Utils, Logs try: @@ -40,10 +41,17 @@ except ImportError: DISTNETCACHE = os.environ.get('DISTNETCACHE', '/tmp/distnetcache') DISTNETSERVER = os.environ.get('DISTNETSERVER', 'http://localhost:8000/cgi-bin/') TARFORMAT = 'w:bz2' -TIMEOUT=60 +TIMEOUT = 60 +REQUIRES = 'requires.txt' re_com = re.compile('\s*#.*', re.M) +def total_version_order(num): + lst = num.split('.') + template = '%10s' * len(lst) + ret = template % tuple(lst) + return ret + def get_distnet_cache(): return getattr(Context.g_module, 'DISTNETCACHE', DISTNETCACHE) @@ -77,8 +85,8 @@ class package(Context.Context): Context.Context.execute(self) pkgfile = send_package_name() if not pkgfile in files: - if not 'requires.txt' in files: - files.append('requires.txt') + if not REQUIRES in files: + files.append(REQUIRES) self.make_tarfile(pkgfile, files, add_to_package=False) def make_tarfile(self, filename, files, **kw): @@ -129,98 +137,219 @@ class publish(Context.Context): if data != 'ok': self.fatal('Could not publish the package %r' % data) +class constraint(object): + def __init__(self, line=''): + self.required_line = line + self.info = [] -class pkg(object): - pass - # name foo - # version 1.0.0 - # required_version 1.0.* - # localfolder /tmp/packages/foo/1.0/ + line = line.strip() + if not line: + return -class package_reader(object): - def read_packages(self, filename='requires.txt'): - self.compute_dependencies(filename) - - def read_package_string(self, txt): - if txt is None: - Logs.error('Hahaha, None!') - self.pkg_list = [] - txt = re.sub(re_com, '', txt) - lines = txt.splitlines() - for line in lines: - if not line: - continue - p = pkg() - p.required_line = line - lst = line.split(',') - p.name = lst[0] - p.requested_version = lst[1] - self.pkg_list.append(p) + lst = line.split(',') + if lst: + self.pkgname = lst[0] + self.required_version = lst[1] for k in lst: a, b, c = k.partition('=') if a and c: - setattr(p, a, c) + self.info.append((a, c)) + def __str__(self): + buf = [] + buf.append(self.pkgname) + buf.append(self.required_version) + for k in self.info: + buf.append('%s=%s' % k) + return ','.join(buf) - def compute_dependencies(self, filename='requires.txt'): + def __repr__(self): + return "requires %s-%s" % (self.pkgname, self.required_version) + + def human_display(self, pkgname, pkgver): + return '%s-%s requires %s-%s' % (pkgname, pkgver, self.pkgname, self.required_version) + + def why(self): + ret = [] + for x in self.info: + if x[0] == 'reason': + ret.append(x[1]) + return ret + + def add_reason(self, reason): + self.info.append(('reason', reason)) + +def parse_constraints(text): + assert(text is not None) + constraints = [] + text = re.sub(re_com, '', text) + lines = text.splitlines() + for line in lines: + line = line.strip() + if not line: + continue + constraints.append(constraint(line)) + return constraints + +def list_package_versions(cachedir, pkgname): + pkgdir = os.path.join(cachedir, pkgname) + try: + versions = os.listdir(pkgdir) + except OSError: + return [] + versions.sort(key=total_version_order) + versions.reverse() + return versions + +class package_reader(Context.Context): + cmd = 'solver' + fun = 'solver' + + def __init__(self, **kw): + Context.Context.__init__(self, **kw) + + self.myproject = getattr(Context.g_module, 'APPNAME', 'project') + self.myversion = getattr(Context.g_module, 'VERSION', '1.0') + self.cache_constraints = {} + self.constraints = [] + + def compute_dependencies(self, filename=REQUIRES): text = Utils.readf(filename) data = safe_urlencode([('text', text)]) - req = Request(get_resolve_url(), data) - try: - response = urlopen(req, timeout=TIMEOUT) - except URLError as e: - Logs.warn('The package server is down! %r' % e) - self.local_resolve(text) + if '--offline' in sys.argv: + self.constraints = self.local_resolve(text) else: - ret = response.read() + req = Request(get_resolve_url(), data) try: - ret = ret.decode('utf-8') - except Exception: - pass - print(ret) - self.read_package_string(ret) + response = urlopen(req, timeout=TIMEOUT) + except URLError as e: + Logs.warn('The package server is down! %r' % e) + self.constraints = self.local_resolve(text) + else: + ret = response.read() + try: + ret = ret.decode('utf-8') + except Exception: + pass + self.trace(ret) + self.constraints = parse_constraints(ret) + self.check_errors() + def check_errors(self): errors = False - for p in self.pkg_list: - if getattr(p, 'error', ''): - Logs.error(p.error) + for c in self.constraints: + if not c.required_version: errors = True + + reasons = c.why() + if len(reasons) == 1: + Logs.error('%s but no matching package could be found in this repository' % reasons[0]) + else: + Logs.error('Conflicts on package %r:' % c.pkgname) + for r in reasons: + Logs.error(' %s' % r) if errors: - raise ValueError('Requirements could not be satisfied!') + self.fatal('The package requirements cannot be satisfied!') + + def load_constraints(self, pkgname, pkgver, requires=REQUIRES): + try: + return self.cache_constraints[(pkgname, pkgver)] + except KeyError: + #Logs.error("no key %r" % (pkgname, pkgver)) + text = Utils.readf(os.path.join(get_distnet_cache(), pkgname, pkgver, requires)) + ret = parse_constraints(text) + self.cache_constraints[(pkgname, pkgver)] = ret + return ret + + def apply_constraint(self, domain, constraint): + vname = constraint.required_version.replace('*', '.*') + rev = re.compile(vname, re.M) + ret = [x for x in domain if rev.match(x)] + return ret + + def trace(self, *k): + if getattr(self, 'debug', None): + Logs.error(*k) + + def solve(self, packages_to_versions={}, packages_to_constraints={}, pkgname='', pkgver='', todo=[], done=[]): + # breadth first search + n_packages_to_versions = dict(packages_to_versions) + n_packages_to_constraints = dict(packages_to_constraints) + + self.trace("calling solve with %r %r %r" % (packages_to_versions, todo, done)) + done = done + [pkgname] + + constraints = self.load_constraints(pkgname, pkgver) + self.trace("constraints %r" % constraints) + + for k in constraints: + try: + domain = n_packages_to_versions[k.pkgname] + except KeyError: + domain = list_package_versions(get_distnet_cache(), k.pkgname) + + + self.trace("constraints?") + if not k.pkgname in done: + todo = todo + [k.pkgname] + + self.trace("domain before %s -> %s, %r" % (pkgname, k.pkgname, domain)) + + # apply the constraint + domain = self.apply_constraint(domain, k) + + self.trace("domain after %s -> %s, %r" % (pkgname, k.pkgname, domain)) + + n_packages_to_versions[k.pkgname] = domain + + # then store the constraint applied + constraints = list(packages_to_constraints.get(k.pkgname, [])) + constraints.append((pkgname, pkgver, k)) + n_packages_to_constraints[k.pkgname] = constraints + + if not domain: + self.trace("no domain while processing constraint %r from %r %r" % (domain, pkgname, pkgver)) + return (n_packages_to_versions, n_packages_to_constraints) + + # next package on the todo list + if not todo: + return (n_packages_to_versions, n_packages_to_constraints) + + n_pkgname = todo[0] + n_pkgver = n_packages_to_versions[n_pkgname][0] + tmp = dict(n_packages_to_versions) + tmp[n_pkgname] = [n_pkgver] + + self.trace("fixed point %s" % n_pkgname) + + return self.solve(tmp, n_packages_to_constraints, n_pkgname, n_pkgver, todo[1:], done) def get_results(self): - buf = [] - for x in self.pkg_list: - buf.append('%s,%s' % (x.name, x.requested_version)) - for y in ('error', 'version'): - if hasattr(x, y): - buf.append(',%s=%s' % (y, getattr(x, y))) - buf.append('\n') - return ''.join(buf) + return '\n'.join([str(c) for c in self.constraints]) + + def solution_to_constraints(self, versions, constraints): + solution = [] + for p in versions.keys(): + c = constraint() + solution.append(c) + + c.pkgname = p + if versions[p]: + c.required_version = versions[p][0] + else: + c.required_version = '' + for (from_pkgname, from_pkgver, c2) in constraints.get(p, ''): + c.add_reason(c2.human_display(from_pkgname, from_pkgver)) + return solution def local_resolve(self, text): - self.read_package_string(text) - for p in self.pkg_list: + self.cache_constraints[(self.myproject, self.myversion)] = parse_constraints(text) + p2v = OrderedDict({self.myproject: [self.myversion]}) + (versions, constraints) = self.solve(p2v, {}, self.myproject, self.myversion, []) + return self.solution_to_constraints(versions, constraints) - pkgdir = os.path.join(get_distnet_cache(), p.name) - try: - versions = os.listdir(pkgdir) - except OSError: - p.error = 'Directory %r does not exist' % pkgdir - continue - - vname = p.requested_version.replace('*', '.*') - rev = re.compile(vname, re.M) - versions = [x for x in versions if rev.match(x)] - versions.sort() - - try: - p.version = versions[0] - except IndexError: - p.error = 'There is no package that satisfies %r %r' % (p.name, p.requested_version) - - def download_to_file(self, p, subdir, tmp): - data = safe_urlencode([('pkgname', p.name), ('pkgver', p.version), ('pkgfile', subdir)]) + def download_to_file(self, pkgname, pkgver, subdir, tmp): + data = safe_urlencode([('pkgname', pkgname), ('pkgver', pkgver), ('pkgfile', subdir)]) req = urlopen(get_download_url(), data, timeout=TIMEOUT) with open(tmp, 'wb') as f: while True: @@ -241,20 +370,21 @@ class package_reader(object): except Exception: pass - def get_pkg_dir(self, pkg, subdir): - pkgdir = os.path.join(get_distnet_cache(), pkg.name, pkg.version) + def get_pkg_dir(self, pkgname, pkgver, subdir): + pkgdir = os.path.join(get_distnet_cache(), pkgname, pkgver) if not os.path.isdir(pkgdir): os.makedirs(pkgdir) target = os.path.join(pkgdir, subdir) + if os.path.exists(target): return target (fd, tmp) = tempfile.mkstemp(dir=pkgdir) try: os.close(fd) - self.download_to_file(pkg, subdir, tmp) - if subdir == 'requires.txt': + self.download_to_file(pkgname, pkgver, subdir, tmp) + if subdir == REQUIRES: os.rename(tmp, target) else: self.extract_tar(subdir, pkgdir, tmp) @@ -267,28 +397,32 @@ class package_reader(object): return target def __iter__(self): - if not hasattr(self, 'pkg_list'): - self.read_packages() + if not self.constraints: self.compute_dependencies() - for x in self.pkg_list: + for x in self.constraints: + if x.pkgname == self.myproject: + continue yield x raise StopIteration + def execute(self): + self.compute_dependencies() + packages = package_reader() def load_tools(ctx, extra): global packages - for pkg in packages: - packages.get_pkg_dir(pkg, extra) - noarchdir = packages.get_pkg_dir(pkg, 'noarch') - #sys.path.append(noarchdir) + for c in packages: + packages.get_pkg_dir(c.pkgname, c.required_version, extra) + noarchdir = packages.get_pkg_dir(c.pkgname, c.required_version, 'noarch') for x in os.listdir(noarchdir): if x.startswith('waf_') and x.endswith('.py'): - ctx.load(x.rstrip('.py'), tooldir=noarchdir) + ctx.load([x.rstrip('.py')], tooldir=[noarchdir]) def options(opt): - packages.read_packages() - load_tools(opt, 'requires.txt') + opt.add_option('--offline', action='store_true') + packages.execute() + load_tools(opt, REQUIRES) def configure(conf): load_tools(conf, conf.variant)