mirror of https://gitlab.com/ita1024/waf.git
wafcache: add support for MinIO/mc cache
Add support for MinIO object storage (https://min.io/) using the MinIO client (https://github.com/minio/mc) to wafcache. MinIO is an open-source, self-hostable, S3 compatible cache. The MinIO client supports MinIO connections as well as normal S3/GCS storages by configuring aliases beforehand. Hint: some distributions have `mc` (the GNU Midnight Commander) installed which is not the minio client, be aware of this (or your build may get stuck with waf waiting for `mc` to never finish)
This commit is contained in:
parent
9cd8a801c7
commit
d1eb41f944
|
@ -17,9 +17,10 @@ The following environment variables may be set:
|
||||||
export WAFCACHE=http://localhost:8080/files/
|
export WAFCACHE=http://localhost:8080/files/
|
||||||
in that case, GET/POST requests are made to urls of the form
|
in that case, GET/POST requests are made to urls of the form
|
||||||
http://localhost:8080/files/000000000/0 (cache management is then up to the server)
|
http://localhost:8080/files/000000000/0 (cache management is then up to the server)
|
||||||
- GCS or S3 bucket
|
- GCS or S3 or MINIO bucket
|
||||||
gs://my-bucket/
|
gs://my-bucket/ (uses gsutil command line tool)
|
||||||
s3://my-bucket/
|
s3://my-bucket/ (uses aws command line tool)
|
||||||
|
minio://my-bucket/ (uses mc command line tool)
|
||||||
* WAFCACHE_NO_PUSH: if set, disables pushing to the cache
|
* WAFCACHE_NO_PUSH: if set, disables pushing to the cache
|
||||||
* WAFCACHE_VERBOSITY: if set, displays more detailed cache operations
|
* WAFCACHE_VERBOSITY: if set, displays more detailed cache operations
|
||||||
|
|
||||||
|
@ -452,8 +453,10 @@ class bucket_cache(object):
|
||||||
def bucket_copy(self, source, target):
|
def bucket_copy(self, source, target):
|
||||||
if CACHE_DIR.startswith('s3://'):
|
if CACHE_DIR.startswith('s3://'):
|
||||||
cmd = ['aws', 's3', 'cp', source, target]
|
cmd = ['aws', 's3', 'cp', source, target]
|
||||||
else:
|
elif CACHE_DIR.startswith('gs://'):
|
||||||
cmd = ['gsutil', 'cp', source, target]
|
cmd = ['gsutil', 'cp', source, target]
|
||||||
|
else:
|
||||||
|
cmd = ['mc', 'cp', source, target]
|
||||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
out, err = proc.communicate()
|
out, err = proc.communicate()
|
||||||
if proc.returncode:
|
if proc.returncode:
|
||||||
|
@ -511,7 +514,9 @@ def loop(service):
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
if CACHE_DIR.startswith('s3://') or CACHE_DIR.startswith('gs://'):
|
if CACHE_DIR.startswith('s3://') or CACHE_DIR.startswith('gs://') or CACHE_DIR.startswith('minio://'):
|
||||||
|
if CACHE_DIR.startswith('minio://'):
|
||||||
|
CACHE_DIR = CACHE_DIR[8:] # minio doesn't need the protocol part, uses config aliases
|
||||||
service = bucket_cache()
|
service = bucket_cache()
|
||||||
elif CACHE_DIR.startswith('http'):
|
elif CACHE_DIR.startswith('http'):
|
||||||
service = netcache()
|
service = netcache()
|
||||||
|
|
Loading…
Reference in New Issue