waf/waflib/extras/remote.py

328 lines
9.6 KiB
Python
Raw Normal View History

2014-01-25 22:36:46 +01:00
#!/usr/bin/env python
# encoding: utf-8
# Remote Builds tool using rsync+ssh
__author__ = "Jérôme Carretero <cJ-waf@zougloub.eu>"
__copyright__ = "Jérôme Carretero, 2013"
"""
Simple Remote Builds
********************
This tool is an *experimental* tool (meaning, do not even try to pollute
the waf bug tracker with bugs in here, contact me directly) providing simple
remote builds.
It uses rsync and ssh to perform the remote builds.
It is intended for performing cross-compilation on platforms where
a cross-compiler is either unavailable (eg. MacOS, QNX) a specific product
does not exist (eg. Windows builds using Visual Studio) or simply not installed.
This tool sends the sources and the waf script to the remote host,
and commands the usual waf execution.
There are alternatives to using this tool, such as setting up shared folders,
logging on to remote machines, and building on the shared folders.
Electing one method or another depends on the size of the program.
Usage
=====
1. Set your wscript file so it includes a list of variants,
e.g.::
from waflib import Utils
top = '.'
out = 'build'
variants = [
'linux_64_debug',
'linux_64_release',
'linux_32_debug',
'linux_32_release',
]
from waflib.extras import remote
def options(opt):
# normal stuff from here on
opt.load('compiler_c')
def configure(conf):
if not conf.variant:
return
# normal stuff from here on
conf.load('compiler_c')
def build(bld):
if not bld.variant:
return
# normal stuff from here on
bld(features='c cprogram', target='app', source='main.c')
2. Build the waf file, so it includes this tool, and put it in the current
directory
.. code:: bash
./waf-light --tools=remote
3. Set the host names to access the hosts:
2014-01-25 22:36:46 +01:00
.. code:: bash
export REMOTE_QNX=user@kiunix
4. Setup the ssh server and ssh keys
2017-02-16 07:03:43 +01:00
The ssh key should not be protected by a password, or it will prompt for it every time.
Create the key on the client:
.. code:: bash
ssh-keygen -t rsa -f foo.rsa
Then copy foo.rsa.pub to the remote machine (user@kiunix:/home/user/.ssh/authorized_keys),
and make sure the permissions are correct (chmod go-w ~ ~/.ssh ~/.ssh/authorized_keys)
A separate key for the build processes can be set in the environment variable WAF_SSH_KEY.
The tool will then use 'ssh-keyscan' to avoid prompting for remote hosts, so
be warned to use this feature on internal networks only (MITM).
.. code:: bash
export WAF_SSH_KEY=~/foo.rsa
2014-01-25 22:36:46 +01:00
5. Perform the build:
.. code:: bash
waf configure_all build_all --remote
"""
import getpass, os, re, sys
2014-01-25 22:36:46 +01:00
from collections import OrderedDict
from waflib import Context, Options, Utils, ConfigSet
from waflib.Build import BuildContext, CleanContext, InstallContext, UninstallContext
from waflib.Configure import ConfigurationContext
is_remote = False
if '--remote' in sys.argv:
is_remote = True
sys.argv.remove('--remote')
class init(Context.Context):
"""
Generates the *_all commands
"""
cmd = 'init'
fun = 'init'
def execute(self):
for x in list(Context.g_module.variants):
self.make_variant(x)
lst = ['remote']
for k in Options.commands:
if k.endswith('_all'):
name = k.replace('_all', '')
2014-01-25 22:36:46 +01:00
for x in Context.g_module.variants:
lst.append('%s_%s' % (name, x))
else:
lst.append(k)
del Options.commands[:]
Options.commands += lst
2014-01-25 22:36:46 +01:00
def make_variant(self, x):
for y in (BuildContext, CleanContext, InstallContext, UninstallContext):
name = y.__name__.replace('Context','').lower()
class tmp(y):
cmd = name + '_' + x
fun = 'build'
variant = x
class tmp(ConfigurationContext):
cmd = 'configure_' + x
fun = 'configure'
variant = x
2014-01-25 22:36:46 +01:00
def __init__(self, **kw):
ConfigurationContext.__init__(self, **kw)
self.setenv(x)
class remote(BuildContext):
cmd = 'remote'
fun = 'build'
def get_ssh_hosts(self):
lst = []
for v in Context.g_module.variants:
self.env.HOST = self.login_to_host(self.variant_to_login(v))
cmd = Utils.subst_vars('${SSH_KEYSCAN} -t rsa,ecdsa ${HOST}', self.env)
out, err = self.cmd_and_log(cmd, output=Context.BOTH, quiet=Context.BOTH)
lst.append(out.strip())
return lst
def setup_private_ssh_key(self):
"""
When WAF_SSH_KEY points to a private key, a .ssh directory will be created in the build directory
Make sure that the ssh key does not prompt for a password
"""
key = os.environ.get('WAF_SSH_KEY', '')
if not key:
return
if not os.path.isfile(key):
self.fatal('Key in WAF_SSH_KEY must point to a valid file')
self.ssh_dir = os.path.join(self.path.abspath(), 'build', '.ssh')
self.ssh_hosts = os.path.join(self.ssh_dir, 'known_hosts')
self.ssh_key = os.path.join(self.ssh_dir, os.path.basename(key))
self.ssh_config = os.path.join(self.ssh_dir, 'config')
for x in self.ssh_hosts, self.ssh_key, self.ssh_config:
if not os.path.isfile(x):
if not os.path.isdir(self.ssh_dir):
os.makedirs(self.ssh_dir)
Utils.writef(self.ssh_key, Utils.readf(key), 'wb')
os.chmod(self.ssh_key, 448)
Utils.writef(self.ssh_hosts, '\n'.join(self.get_ssh_hosts()))
os.chmod(self.ssh_key, 448)
Utils.writef(self.ssh_config, 'UserKnownHostsFile %s' % self.ssh_hosts, 'wb')
os.chmod(self.ssh_config, 448)
self.env.SSH_OPTS = ['-F', self.ssh_config, '-i', self.ssh_key]
self.env.append_value('RSYNC_SEND_OPTS', '--exclude=build/.ssh')
2014-01-25 22:36:46 +01:00
def skip_unbuildable_variant(self):
# skip variants that cannot be built on this OS
for k in Options.commands:
a, _, b = k.partition('_')
if b in Context.g_module.variants:
c, _, _ = b.partition('_')
if c != Utils.unversioned_sys_platform():
Options.commands.remove(k)
def login_to_host(self, login):
return re.sub(r'(\w+@)', '', login)
def variant_to_login(self, variant):
"""linux_32_debug -> search env.LINUX_32 and then env.LINUX"""
2014-01-25 22:36:46 +01:00
x = variant[:variant.rfind('_')]
ret = os.environ.get('REMOTE_' + x.upper(), '')
if not ret:
x = x[:x.find('_')]
ret = os.environ.get('REMOTE_' + x.upper(), '')
if not ret:
ret = '%s@localhost' % getpass.getuser()
return ret
2014-01-25 22:36:46 +01:00
def execute(self):
global is_remote
if not is_remote:
self.skip_unbuildable_variant()
else:
BuildContext.execute(self)
def restore(self):
self.top_dir = os.path.abspath(Context.g_module.top)
self.srcnode = self.root.find_node(self.top_dir)
self.path = self.srcnode
self.out_dir = os.path.join(self.top_dir, Context.g_module.out)
self.bldnode = self.root.make_node(self.out_dir)
self.bldnode.mkdir()
self.env = ConfigSet.ConfigSet()
def extract_groups_of_builds(self):
"""Return a dict mapping each variants to the commands to build"""
self.vgroups = {}
2014-01-25 22:36:46 +01:00
for x in reversed(Options.commands):
_, _, variant = x.partition('_')
if variant in Context.g_module.variants:
try:
dct = self.vgroups[variant]
2014-01-25 22:36:46 +01:00
except KeyError:
dct = self.vgroups[variant] = OrderedDict()
2014-01-25 22:36:46 +01:00
try:
dct[variant].append(x)
except KeyError:
dct[variant] = [x]
Options.commands.remove(x)
def custom_options(self, login):
try:
return Context.g_module.host_options[login]
except (AttributeError, KeyError):
return {}
def recurse(self, *k, **kw):
self.env.RSYNC = getattr(Context.g_module, 'rsync', 'rsync -a --chmod=u+rwx')
self.env.SSH = getattr(Context.g_module, 'ssh', 'ssh')
self.env.SSH_KEYSCAN = getattr(Context.g_module, 'ssh_keyscan', 'ssh-keyscan')
2014-01-25 22:36:46 +01:00
try:
self.env.WAF = getattr(Context.g_module, 'waf')
except AttributeError:
try:
os.stat('waf')
except KeyError:
self.fatal('Put a waf file in the directory (./waf-light --tools=remote)')
else:
self.env.WAF = './waf'
self.extract_groups_of_builds()
self.setup_private_ssh_key()
for k, v in self.vgroups.items():
2014-01-25 22:36:46 +01:00
task = self(rule=rsync_and_ssh, always=True)
task.env.login = self.variant_to_login(k)
2014-01-25 22:36:46 +01:00
task.env.commands = []
for opt, value in v.items():
task.env.commands += value
task.env.variant = task.env.commands[0].partition('_')[2]
for opt, value in self.custom_options(k):
task.env[opt] = value
self.jobs = len(self.vgroups)
2014-01-25 22:36:46 +01:00
def make_mkdir_command(self, task):
return Utils.subst_vars('${SSH} ${SSH_OPTS} ${login} "rm -fr ${remote_dir} && mkdir -p ${remote_dir}"', task.env)
2014-01-25 22:36:46 +01:00
def make_send_command(self, task):
return Utils.subst_vars('${RSYNC} ${RSYNC_SEND_OPTS} -e "${SSH} ${SSH_OPTS}" ${local_dir} ${login}:${remote_dir}', task.env)
2014-01-25 22:36:46 +01:00
def make_exec_command(self, task):
txt = '''${SSH} ${SSH_OPTS} ${login} "cd ${remote_dir} && ${WAF} ${commands}"'''
2014-01-25 22:36:46 +01:00
return Utils.subst_vars(txt, task.env)
def make_save_command(self, task):
return Utils.subst_vars('${RSYNC} ${RSYNC_SAVE_OPTS} -e "${SSH} ${SSH_OPTS}" ${login}:${remote_dir_variant} ${build_dir}', task.env)
2014-01-25 22:36:46 +01:00
def rsync_and_ssh(task):
# remove a warning
task.uid_ = id(task)
2014-01-25 22:36:46 +01:00
bld = task.generator.bld
task.env.user, _, _ = task.env.login.partition('@')
task.env.hdir = Utils.to_hex(Utils.h_list((task.generator.path.abspath(), task.env.variant)))
task.env.remote_dir = '~%s/wafremote/%s' % (task.env.user, task.env.hdir)
task.env.local_dir = bld.srcnode.abspath() + '/'
task.env.remote_dir_variant = '%s/%s/%s' % (task.env.remote_dir, Context.g_module.out, task.env.variant)
task.env.build_dir = bld.bldnode.abspath()
ret = task.exec_command(bld.make_mkdir_command(task))
if ret:
return ret
ret = task.exec_command(bld.make_send_command(task))
if ret:
return ret
ret = task.exec_command(bld.make_exec_command(task))
if ret:
return ret
ret = task.exec_command(bld.make_save_command(task))
if ret:
return ret