mirror of
https://gitlab.com/ita1024/waf.git
synced 2024-11-22 09:57:15 +01:00
Issue 1138, Issue 1139, Issue 1141
This commit is contained in:
Thomas Nagy
parent
37854f57a2
commit
9004b1fd95
142
waflib/extras/run_do_script.py
Normal file
142
waflib/extras/run_do_script.py
Normal file
@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
# Hans-Martin von Gaudecker, 2012
|
||||
|
||||
"""
|
||||
Run a Stata do-file in the directory specified by **ctx.bldnode**. The
|
||||
first and only argument will be the name of the do-file (no extension),
|
||||
which can be accessed inside the do-file by the local macro `1'. Useful
|
||||
for keeping a log file.
|
||||
|
||||
The tool uses the log-file that is automatically kept by Stata only
|
||||
for error-catching purposes, it will be destroyed if the task finished
|
||||
without error. In case of an error, you can inspect it as **do_file.log**
|
||||
in the **ctx.bldnode** directory.
|
||||
|
||||
Note that Stata will not return an error code if it exits abnormally --
|
||||
catching errors relies on parsing the log-file mentioned before. Should
|
||||
the parser behave incorrectly please send an email to hmgaudecker [at] gmail.
|
||||
|
||||
**Careful**
|
||||
|
||||
The tool will not work if multiple do-files of the same name---but in
|
||||
different directories---are run at the same time! Avoid this situation.
|
||||
|
||||
Usage:
|
||||
|
||||
ctx(features='run_do_file',
|
||||
source='some_file.do',
|
||||
target=['some_table.tex', 'some_figure.eps'],
|
||||
deps='some_data.csv')
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import os, re, sys
|
||||
from waflib import Task, TaskGen, Logs
|
||||
|
||||
if sys.platform == 'darwin':
|
||||
STATA_COMMANDS = ['Stata64MP', 'StataMP',
|
||||
'Stata64SE', 'StataSE',
|
||||
'Stata64', 'Stata']
|
||||
STATAFLAGS = '-e -q do'
|
||||
STATAENCODING = 'MacRoman'
|
||||
elif sys.platform.startswith('linux'):
|
||||
STATA_COMMANDS = ['stata-mp', 'stata-se', 'stata']
|
||||
STATAFLAGS = '-b -q do'
|
||||
# Not sure whether this is correct...
|
||||
STATAENCODING = 'Latin-1'
|
||||
elif sys.platform.lower().startswith('win'):
|
||||
STATA_COMMANDS = ['StataMP-64', 'StataMP-ia',
|
||||
'StataMP', 'StataSE-64',
|
||||
'StataSE-ia', 'StataSE',
|
||||
'Stata-64', 'Stata-ia',
|
||||
'Stata.e', 'WMPSTATA',
|
||||
'WSESTATA', 'WSTATA']
|
||||
STATAFLAGS = '/e do'
|
||||
STATAENCODING = 'Latin-1'
|
||||
else:
|
||||
raise Exception("Unknown sys.platform: %s " % sys.platform)
|
||||
|
||||
def configure(ctx):
|
||||
ctx.find_program(STATA_COMMANDS, var='STATACMD', errmsg="""\n
|
||||
No Stata executable found!\n\n
|
||||
If Stata is needed:\n
|
||||
1) Check the settings of your system path.
|
||||
2) Note we are looking for Stata executables called: %s
|
||||
If yours has a different name, please report to hmgaudecker [at] gmail\n
|
||||
Else:\n
|
||||
Do not load the 'run_do_file' tool in the main wscript.\n\n""" % STATA_COMMANDS)
|
||||
ctx.env.STATAFLAGS = STATAFLAGS
|
||||
ctx.env.STATAENCODING = STATAENCODING
|
||||
|
||||
@Task.update_outputs
|
||||
class run_do_file_base(Task.Task):
|
||||
"""Run a Stata do-file from the bldnode directory."""
|
||||
run_str = '"${STATACMD}" ${STATAFLAGS} "${SRC[0].abspath()}" "${DOFILETRUNK}"'
|
||||
shell = True
|
||||
|
||||
class run_do_file(run_do_file_base):
|
||||
"""Use the log-file automatically kept by Stata for error-catching.
|
||||
Erase it if the task finished without error. If not, it will show
|
||||
up as do_file.log in the bldnode directory.
|
||||
"""
|
||||
def run(self):
|
||||
run_do_file_base.run(self)
|
||||
ret, log_tail = self.check_erase_log_file()
|
||||
if ret:
|
||||
Logs.error("""Running Stata on %s failed with code %r.\n\nCheck the log file %s, last 10 lines\n\n%s\n\n\n""" % (
|
||||
self.inputs[0].nice_path(), ret, self.env.LOGFILEPATH, log_tail))
|
||||
return ret
|
||||
|
||||
def check_erase_log_file(self):
|
||||
"""Parse Stata's default log file and erase it if everything okay.
|
||||
|
||||
Parser is based on Brendan Halpin's shell script found here:
|
||||
http://teaching.sociology.ul.ie/bhalpin/wordpress/?p=122
|
||||
"""
|
||||
|
||||
if sys.version_info.major >= 3:
|
||||
kwargs = {'file': self.env.LOGFILEPATH, 'mode': 'r', 'encoding': self.env.STATAENCODING}
|
||||
else:
|
||||
kwargs = {'name': self.env.LOGFILEPATH, 'mode': 'r'}
|
||||
|
||||
with open(**kwargs) as log:
|
||||
log_tail = log.readlines()[-10:]
|
||||
for line in log_tail:
|
||||
error_found = re.match("r\(([0-9]+)\)", line)
|
||||
if error_found:
|
||||
return error_found.group(1), ''.join(log_tail)
|
||||
else:
|
||||
pass
|
||||
# Only end up here if the parser did not identify an error.
|
||||
os.remove(self.env.LOGFILEPATH)
|
||||
return None, None
|
||||
|
||||
|
||||
@TaskGen.feature('run_do_file')
|
||||
@TaskGen.before_method('process_source')
|
||||
def apply_run_do_file(tg):
|
||||
"""Task generator customising the options etc. to call Stata in batch
|
||||
mode for running a do-file.
|
||||
"""
|
||||
|
||||
# Convert sources and targets to nodes
|
||||
src_node = tg.path.find_resource(tg.source)
|
||||
tgt_nodes = [tg.path.find_or_declare(t) for t in tg.to_list(tg.target)]
|
||||
|
||||
tsk = tg.create_task('run_do_file', src=src_node, tgt=tgt_nodes)
|
||||
tsk.env.DOFILETRUNK = os.path.splitext(src_node.name)[0]
|
||||
tsk.env.LOGFILEPATH = os.path.join(tg.bld.bldnode.abspath(), '%s.log' % (tsk.env.DOFILETRUNK))
|
||||
|
||||
# dependencies (if the attribute 'deps' changes, trigger a recompilation)
|
||||
for x in tg.to_list(getattr(tg, 'deps', [])):
|
||||
node = tg.path.find_resource(x)
|
||||
if not node:
|
||||
tg.bld.fatal('Could not find dependency %r for running %r' % (x, src_node.nice_path()))
|
||||
tsk.dep_nodes.append(node)
|
||||
Logs.debug('deps: found dependencies %r for running %r' % (tsk.dep_nodes, src_node.nice_path()))
|
||||
|
||||
# Bypass the execution of process_source by setting the source to an empty list
|
||||
tg.source = []
|
||||
|
||||
110
waflib/extras/run_py_script.py
Normal file
110
waflib/extras/run_py_script.py
Normal file
@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
# Hans-Martin von Gaudecker, 2012
|
||||
|
||||
"""
|
||||
Run a Python script in the directory specified by **ctx.bldnode**.
|
||||
|
||||
Select a Python version by specifying the **version** keyword for
|
||||
the task generator instance as integer 2 or 3. Default is 3.
|
||||
|
||||
If the build environment has an attribute "PROJECT_PATHS" with
|
||||
a key "PROJECT_ROOT", its value will be appended to the PYTHONPATH.
|
||||
Same a string passed to the optional **add_to_pythonpath**
|
||||
keyword (appended after the PROJECT_ROOT).
|
||||
|
||||
Usage:
|
||||
|
||||
ctx(features='run_py_script', version=3,
|
||||
source='some_script.py',
|
||||
target=['some_table.tex', 'some_figure.eps'],
|
||||
deps='some_data.csv',
|
||||
add_to_pythonpath='src/some/library')
|
||||
|
||||
"""
|
||||
|
||||
import os, re
|
||||
from waflib import Task, TaskGen, Logs
|
||||
|
||||
|
||||
def configure(ctx):
|
||||
"""TODO: Needs updating for Windows once
|
||||
"PEP 397":http://www.python.org/dev/peps/pep-0397/ is settled.
|
||||
|
||||
"""
|
||||
# Counter how many Python interpreters are found
|
||||
cnt = 0
|
||||
conf.find_program('python', var='PY2CMD', mandatory=False)
|
||||
conf.find_program('python3', var='PY3CMD', mandatory=False)
|
||||
if not conf.env.PY2CMD and not conf.env.PY3CMD:
|
||||
ctx.fatal("No Python interpreter found!")
|
||||
|
||||
@Task.update_outputs
|
||||
class run_py_2_script(Task.Task):
|
||||
"""Run a Python 2 script."""
|
||||
run_str = '${PY2CMD} ${SRC[0].abspath()}'
|
||||
shell=True
|
||||
|
||||
@Task.update_outputs
|
||||
class run_py_3_script(Task.Task):
|
||||
"""Run a Python 3 script."""
|
||||
run_str = '${PY3CMD} ${SRC[0].abspath()}'
|
||||
shell=True
|
||||
|
||||
@TaskGen.feature('run_py_script')
|
||||
@TaskGen.before_method('process_source')
|
||||
def apply_run_py_script(tg):
|
||||
"""Task generator for running either Python 2 or Python 3 on a single
|
||||
script.
|
||||
|
||||
Attributes:
|
||||
|
||||
* source -- A **single** source node or string. (required)
|
||||
* target -- A single target or list of targets (nodes or strings).
|
||||
* deps -- A single dependency or list of dependencies (nodes or strings)
|
||||
* add_to_pythonpath -- A string that will be appended to the PYTHONPATH environment variable.
|
||||
|
||||
If the build environment has an attribute "PROJECT_PATHS" with
|
||||
a key "PROJECT_ROOT", its value will be appended to the PYTHONPATH.
|
||||
|
||||
"""
|
||||
|
||||
# Set the Python version to use, default to 3.
|
||||
v = getattr(tg, 'version', 3)
|
||||
if v not in (2, 3): raise ValueError("Specify the 'version' attribute for run_py_script task generator as integer 2 or 3.\n Got: %s" %v)
|
||||
|
||||
# Convert sources and targets to nodes
|
||||
src_node = tg.path.find_resource(tg.source)
|
||||
tgt_nodes = [tg.path.find_or_declare(t) for t in tg.to_list(tg.target)]
|
||||
|
||||
# Create the task.
|
||||
tsk = tg.create_task('run_py_%d_script' %v, src=src_node, tgt=tgt_nodes)
|
||||
|
||||
# custom execution environment
|
||||
# TODO use a list and os.sep.join(lst) at the end instead of concatenating strings
|
||||
tsk.env.env = dict(os.environ)
|
||||
tsk.env.env['PYTHONPATH'] = tsk.env.env.get('PYTHONPATH', '')
|
||||
project_paths = getattr(tsk.env, 'PROJECT_PATHS', None)
|
||||
if project_paths and 'PROJECT_ROOT' in project_paths:
|
||||
tsk.env.env['PYTHONPATH'] += os.pathsep + project_paths['PROJECT_ROOT'].abspath()
|
||||
if getattr(tg, 'add_to_pythonpath', None):
|
||||
tsk.env.env['PYTHONPATH'] += os.pathsep + tg.add_to_pythonpath
|
||||
|
||||
# Clean up the PYTHONPATH -- replace double occurrences of path separator
|
||||
tsk.env.env['PYTHONPATH'] = re.sub(os.pathsep + '+', os.pathsep, tsk.env.env['PYTHONPATH'])
|
||||
|
||||
# Clean up the PYTHONPATH -- doesn't like starting with path separator
|
||||
if tsk.env.env['PYTHONPATH'].startswith(os.pathsep):
|
||||
tsk.env.env['PYTHONPATH'] = tsk.env.env['PYTHONPATH'][1:]
|
||||
|
||||
# dependencies (if the attribute 'deps' changes, trigger a recompilation)
|
||||
for x in tg.to_list(getattr(tg, 'deps', [])):
|
||||
node = tg.path.find_resource(x)
|
||||
if not node:
|
||||
tg.bld.fatal('Could not find dependency %r for running %r' % (x, src_node.nice_path()))
|
||||
tsk.dep_nodes.append(node)
|
||||
Logs.debug('deps: found dependencies %r for running %r' % (tsk.dep_nodes, src_node.nice_path()))
|
||||
|
||||
# Bypass the execution of process_source by setting the source to an empty list
|
||||
tg.source = []
|
||||
|
||||
88
waflib/extras/run_r_script.py
Normal file
88
waflib/extras/run_r_script.py
Normal file
@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
# Hans-Martin von Gaudecker, 2012
|
||||
|
||||
"""
|
||||
Run a R script in the directory specified by **ctx.bldnode**.
|
||||
|
||||
For error-catching purposes, keep an own log-file that is destroyed if the
|
||||
task finished without error. If not, it will show up as rscript_[index].log
|
||||
in the bldnode directory.
|
||||
|
||||
Usage:
|
||||
|
||||
ctx(features='run_r_script',
|
||||
source='some_script.r',
|
||||
target=['some_table.tex', 'some_figure.eps'],
|
||||
deps='some_data.csv')
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import os, re, sys
|
||||
from waflib import Task, TaskGen, Logs
|
||||
|
||||
R_COMMANDS = ['RTerm', 'R', 'r']
|
||||
|
||||
def configure(ctx):
|
||||
ctx.find_program(R_COMMANDS, var='RCMD', errmsg = """\n
|
||||
No R executable found!\n\n
|
||||
If R is needed:\n
|
||||
1) Check the settings of your system path.
|
||||
2) Note we are looking for R executables called: %s
|
||||
If yours has a different name, please report to hmgaudecker [at] gmail\n
|
||||
Else:\n
|
||||
Do not load the 'run_r_script' tool in the main wscript.\n\n""" % R_COMMANDS)
|
||||
ctx.env.RFLAGS = 'CMD BATCH --slave'
|
||||
|
||||
@Task.update_outputs
|
||||
class run_r_script_base(Task.Task):
|
||||
"""Run a R script."""
|
||||
run_str = '"${RCMD}" ${RFLAGS} "${SRC[0].abspath()}" "${LOGFILEPATH}"'
|
||||
shell = True
|
||||
|
||||
class run_r_script(run_r_script_base):
|
||||
"""Erase the R overall log file if everything went okay, else raise an
|
||||
error and print its 10 last lines.
|
||||
"""
|
||||
def run(self):
|
||||
ret = run_r_script_base.run(self)
|
||||
logfile = self.env.LOGFILEPATH
|
||||
if ret:
|
||||
mode = 'r'
|
||||
if sys.version_info.major >= 3:
|
||||
mode = 'rb'
|
||||
with open(logfile, mode=mode) as f:
|
||||
tail = f.readlines()[-10:]
|
||||
Logs.error("""Running R on %s returned the error %r\n\nCheck the log file %s, last 10 lines\n\n%s\n\n\n""" % (
|
||||
self.inputs[0].nice_path(), ret, logfile, '\n'.join(tail)))
|
||||
else:
|
||||
os.remove(logfile)
|
||||
return ret
|
||||
|
||||
|
||||
@TaskGen.feature('run_r_script')
|
||||
@TaskGen.before_method('process_source')
|
||||
def apply_run_r_script(tg):
|
||||
"""Task generator customising the options etc. to call R in batch
|
||||
mode for running a R script.
|
||||
"""
|
||||
|
||||
# Convert sources and targets to nodes
|
||||
src_node = tg.path.find_resource(tg.source)
|
||||
tgt_nodes = [tg.path.find_or_declare(t) for t in tg.to_list(tg.target)]
|
||||
|
||||
tsk = tg.create_task('run_r_script', src=src_node, tgt=tgt_nodes)
|
||||
tsk.env.LOGFILEPATH = os.path.join(tg.bld.bldnode.abspath(), '%s_%d.log' % (os.path.splitext(src_node.name)[0], tg.idx))
|
||||
|
||||
# dependencies (if the attribute 'deps' changes, trigger a recompilation)
|
||||
for x in tg.to_list(getattr(tg, 'deps', [])):
|
||||
node = tg.path.find_resource(x)
|
||||
if not node:
|
||||
tg.bld.fatal('Could not find dependency %r for running %r' % (x, src_node.nice_path()))
|
||||
tsk.dep_nodes.append(node)
|
||||
Logs.debug('deps: found dependencies %r for running %r' % (tsk.dep_nodes, src_node.nice_path()))
|
||||
|
||||
# Bypass the execution of process_source by setting the source to an empty list
|
||||
tg.source = []
|
||||
|
||||
Loading…
Reference in New Issue
Block a user