FWGS/waf
2
0
Fork 0
mirror of https://gitlab.com/ita1024/waf.git synced 2024-11-25 11:19:52 +01:00
Code Issues Projects Releases Wiki Activity

Improve WAFCACHE_CMD parameter handling

Browse Source
This commit is contained in:
Thomas Nagy 2020-11-03 22:19:13 +01:00
parent 5584d1e687
commit 4e61e3a563
1 changed files with 19 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
waflib/Tools/wafcache.py
View File

@ -21,6 +21,8 @@ The following environment variables may be set:
gs://my-bucket/ (uses gsutil command line tool or WAFCACHE_CMD) gs://my-bucket/ (uses gsutil command line tool or WAFCACHE_CMD)
s3://my-bucket/ (uses aws command line tool or WAFCACHE_CMD) s3://my-bucket/ (uses aws command line tool or WAFCACHE_CMD)
minio://my-bucket/ (uses mc command line tool or WAFCACHE_CMD) minio://my-bucket/ (uses mc command line tool or WAFCACHE_CMD)
* WAFCACHE_CMD: custom upload/download command, for example:
WAFCACHE_CMD="gsutil cp %{SRC} %{TGT}"
* WAFCACHE_NO_PUSH: if set, disables pushing to the cache * WAFCACHE_NO_PUSH: if set, disables pushing to the cache
* WAFCACHE_VERBOSITY: if set, displays more detailed cache operations * WAFCACHE_VERBOSITY: if set, displays more detailed cache operations
@ -32,11 +34,6 @@ File cache specific options:
* WAFCACHE_EVICT_INTERVAL_MINUTES: minimum time interval to try * WAFCACHE_EVICT_INTERVAL_MINUTES: minimum time interval to try
and trim the cache (3 minutess) and trim the cache (3 minutess)
Bucket cache specific options:
* WAFCACHE_CMD: command to use to access bucket cache. It must contain
exactly two %s, assuming first is substituted by source
and second by target (see defaults for examples)
Usage:: Usage::
def build(bld): def build(bld):
@ -48,7 +45,7 @@ To troubleshoot::
waf clean build --zones=wafcache waf clean build --zones=wafcache
""" """
import atexit, base64, errno, fcntl, getpass, os, shutil, sys, time, traceback, urllib3, shlex import atexit, base64, errno, fcntl, getpass, os, re, shutil, sys, time, traceback, urllib3, shlex
try: try:
import subprocess32 as subprocess import subprocess32 as subprocess
except ImportError: except ImportError:
@ -60,6 +57,7 @@ if not os.path.isdir(base_cache):
default_wafcache_dir = os.path.join(base_cache, 'wafcache_' + getpass.getuser()) default_wafcache_dir = os.path.join(base_cache, 'wafcache_' + getpass.getuser())
CACHE_DIR = os.environ.get('WAFCACHE', default_wafcache_dir) CACHE_DIR = os.environ.get('WAFCACHE', default_wafcache_dir)
WAFCACHE_CMD = os.environ.get('WAFCACHE_CMD')
TRIM_MAX_FOLDERS = int(os.environ.get('WAFCACHE_TRIM_MAX_FOLDER', 1000000)) TRIM_MAX_FOLDERS = int(os.environ.get('WAFCACHE_TRIM_MAX_FOLDER', 1000000))
EVICT_INTERVAL_MINUTES = int(os.environ.get('WAFCACHE_EVICT_INTERVAL_MINUTES', 3)) EVICT_INTERVAL_MINUTES = int(os.environ.get('WAFCACHE_EVICT_INTERVAL_MINUTES', 3))
EVICT_MAX_BYTES = int(os.environ.get('WAFCACHE_EVICT_MAX_BYTES', 10**10)) EVICT_MAX_BYTES = int(os.environ.get('WAFCACHE_EVICT_MAX_BYTES', 10**10))
@ -67,17 +65,7 @@ WAFCACHE_NO_PUSH = 1 if os.environ.get('WAFCACHE_NO_PUSH') else 0
WAFCACHE_VERBOSITY = 1 if os.environ.get('WAFCACHE_VERBOSITY') else 0 WAFCACHE_VERBOSITY = 1 if os.environ.get('WAFCACHE_VERBOSITY') else 0
OK = "ok" OK = "ok"
WAFCACHE_CMD = os.environ.get('WAFCACHE_CMD', None) re_waf_cmd = re.compile('(?P<src>%{SRC})|(?P<tgt>%{TGT})')
# Some defaults if WAFCACHE_CMD is not passed
if not WAFCACHE_CMD:
if CACHE_DIR.startswith('s3://'):
WAFCACHE_CMD = 'aws s3 cp "%s" "%s"'
elif CACHE_DIR.startswith('gs://'):
WAFCACHE_CMD = 'gsutil cp "%s" "%s"'
else:
WAFCACHE_CMD = 'mc cp "%s" "%s"'
try: try:
import cPickle import cPickle
@ -476,7 +464,20 @@ class fcache(object):
class bucket_cache(object): class bucket_cache(object):
def bucket_copy(self, source, target): def bucket_copy(self, source, target):
cmd = shlex.split(WAFCACHE_CMD % (source, target)) if WAFCACHE_CMD:
def replacer(match):
if match.group('src'):
return source
elif match.group('tgt'):
return target
cmd = [re_waf_cmd.sub(replacer, x) for x in shlex.split(WAFCACHE_CMD)]
elif CACHE_DIR.startswith('s3://'):
cmd = ['aws', 's3', 'cp', source, target]
elif CACHE_DIR.startswith('gs://'):
cmd = ['gsutil', 'cp', source, target]
else:
cmd = ['mc', 'cp', source, target]
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = proc.communicate() out, err = proc.communicate()
if proc.returncode: if proc.returncode: