OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity

-----BEGIN PGP SIGNATURE-----

Browse Source 
Version: GnuPG v2
 
 iQEcBAABCAAGBQJXj15cAAoJEMo1YkxqkXHGkwIIAIgXZ7ciQNS6HK8WWlfvulfh
 gFnu32HDNih3zYk6N5NNcpHxi16dYLdj98WteWlkYUwwJ2iQBH8e0VJPVMYzJC+g
 pdbaUjXScpCkumA+vH6PgUjgJwH3Z1FMj+r9I1ZF6POy17DjOy6xmCCr+Pvh0sxm
 NfRzgnUM1nsHvfVS6WM+NorlmEX/wvkWw/qBjv49N5hoJw9I0saJopNM+oh6+Pgy
 A87DM83O0a8fHPBoPV7L6TDYasNl/Y26iCliBu9qxW/pGODjw8ohrQkxq8Bopo08
 jy7ITHNfrcK44PFMmCZELbygtLZe5eB5qmHndyAGQhDjrHpe+/cv84ar/Dh+MrY=
 =PXt6
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/famz/tags/docker-pull-request' into staging

# gpg: Signature made Wed 20 Jul 2016 12:19:56 BST
# gpg:                using RSA key 0xCA35624C6A9171C6
# gpg: Good signature from "Fam Zheng <famz@redhat.com>"
# gpg: WARNING: This key is not certified with a trusted signature!
# gpg:          There is no indication that the signature belongs to the owner.
# Primary key fingerprint: 5003 7CB7 9706 0F76 F021  AD56 CA35 624C 6A91 71C6

* remotes/famz/tags/docker-pull-request:
  docker: pass EXECUTABLE to build script
  docker: Don't start a container that doesn't exist
  docker: Add "images" subcommand to docker.py
  docker: Fix exit code if $CMD failed
  docker: More sensible run script
  tests/docker/docker.py: add update operation
  tests/docker/dockerfiles: new debian-bootstrap.docker
  tests/docker/docker.py: check and run .pre script
  tests/docker/docker.py: support --include-executable
  tests/docker/docker.py: docker_dir outside build

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2016-07-20 18:52:10 +01:00
commit 46ca418d9f
5 changed files with 280 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tests/docker/Makefile.include
View File

@ -46,7 +46,8 @@ docker-image: ${DOCKER_TARGETS}
docker-image-%: $(DOCKER_FILES_DIR)/%.docker
$(call quiet-command,\
$(SRC_PATH)/tests/docker/docker.py build qemu:$* $< \
$(if $V,,--quiet) $(if $(NOCACHE),--no-cache),\
$(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \
$(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\
" BUILD $*")
# Expand all the pre-requistes for each docker image and test combination
@ -95,6 +96,7 @@ docker:
@echo ' DEBUG=1 Stop and drop to shell in the created container'
@echo ' before running the command.'
@echo ' NOCACHE=1 Ignore cache when build images.'
@echo ' EXECUTABLE=<path> Include executable in image.'
docker-run-%: CMD = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\1/')
docker-run-%: IMAGE = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\2/')
@ -105,6 +107,9 @@ docker-run-%: docker-qemu-src
fi
$(if $(filter $(TESTS),$(CMD)),$(if $(filter $(IMAGES),$(IMAGE)), \
$(call quiet-command,\
if $(SRC_PATH)/tests/docker/docker.py images \
--format={{.Repository}}:{{.Tag}} | \
grep -qx qemu:$(IMAGE); then \
$(SRC_PATH)/tests/docker/docker.py run $(if $V,,--rm) \
-t \
$(if $(DEBUG),-i,--net=none) \
@ -114,11 +119,10 @@ docker-run-%: docker-qemu-src
-e CCACHE_DIR=/var/tmp/ccache \
-v $$(realpath $(DOCKER_SRC_COPY)):/var/tmp/qemu:z$(COMMA)ro \
-v $(DOCKER_CCACHE_DIR):/var/tmp/ccache:z \
-w /var/tmp/qemu \
qemu:$(IMAGE) \
$(if $V,/bin/bash -x ,) \
./run \
/var/tmp/qemu/run \
$(CMD); \
fi \
, " RUN $(CMD) in $(IMAGE)")))
docker-clean:

157
tests/docker/docker.py
View File

@ -20,7 +20,10 @@ import atexit
import uuid
import argparse
import tempfile
from shutil import copy
import re
from tarfile import TarFile, TarInfo
from StringIO import StringIO
from shutil import copy, rmtree
def _text_checksum(text):
"""Calculate a digest string unique to the text content"""
@ -38,6 +41,54 @@ def _guess_docker_command():
raise Exception("Cannot find working docker command. Tried:\n%s" % \
commands_txt)
def _copy_with_mkdir(src, root_dir, sub_path):
"""Copy src into root_dir, creating sub_path as needed."""
dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
try:
os.makedirs(dest_dir)
except OSError:
# we can safely ignore already created directories
pass
dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
copy(src, dest_file)
def _get_so_libs(executable):
"""Return a list of libraries associated with an executable.
The paths may be symbolic links which would need to be resolved to
ensure theright data is copied."""
libs = []
ldd_re = re.compile(r"(/.*/)(\S*)")
try:
ldd_output = subprocess.check_output(["ldd", executable])
for line in ldd_output.split("\n"):
search = ldd_re.search(line)
if search and len(search.groups()) == 2:
so_path = search.groups()[0]
so_lib = search.groups()[1]
libs.append("%s/%s" % (so_path, so_lib))
except subprocess.CalledProcessError:
print "%s had no associated libraries (static build?)" % (executable)
return libs
def _copy_binary_with_libs(src, dest_dir):
"""Copy a binary executable and all its dependant libraries.
This does rely on the host file-system being fairly multi-arch
aware so the file don't clash with the guests layout."""
_copy_with_mkdir(src, dest_dir, "/usr/bin")
libs = _get_so_libs(src)
if libs:
for l in libs:
so_path = os.path.dirname(l)
_copy_with_mkdir(l , dest_dir, so_path)
class Docker(object):
""" Running Docker commands """
def __init__(self):
@ -45,9 +96,11 @@ class Docker(object):
self._instances = []
atexit.register(self._kill_instances)
def _do(self, cmd, quiet=True, **kwargs):
def _do(self, cmd, quiet=True, infile=None, **kwargs):
if quiet:
kwargs["stdout"] = subprocess.PIPE
if infile:
kwargs["stdin"] = infile
return subprocess.call(self._command + cmd, **kwargs)
def _do_kill_instances(self, only_known, only_active=True):
@ -87,22 +140,27 @@ class Docker(object):
labels = json.loads(resp)[0]["Config"].get("Labels", {})
return labels.get("com.qemu.dockerfile-checksum", "")
def build_image(self, tag, dockerfile, df_path, quiet=True, argv=None):
def build_image(self, tag, docker_dir, dockerfile, quiet=True, argv=None):
if argv == None:
argv = []
tmp_dir = tempfile.mkdtemp(prefix="docker_build")
tmp_df = tempfile.NamedTemporaryFile(dir=tmp_dir, suffix=".docker")
tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
tmp_df.write(dockerfile)
tmp_df.write("\n")
tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
_text_checksum(dockerfile))
tmp_df.flush()
self._do(["build", "-t", tag, "-f", tmp_df.name] + argv + \
[tmp_dir],
[docker_dir],
quiet=quiet)
def update_image(self, tag, tarball, quiet=True):
"Update a tagged image using "
self._do(["build", "-t", tag, "-"], quiet=quiet, infile=tarball)
def image_matches_dockerfile(self, tag, dockerfile):
try:
checksum = self.get_image_dockerfile_checksum(tag)
@ -121,6 +179,9 @@ class Docker(object):
self._instances.remove(label)
return ret
def command(self, cmd, argv, quiet):
return self._do([cmd] + argv, quiet=quiet)
class SubCommand(object):
"""A SubCommand template base class"""
name = None # Subcommand name
@ -151,6 +212,10 @@ class BuildCommand(SubCommand):
""" Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
name = "build"
def args(self, parser):
parser.add_argument("--include-executable", "-e",
help="""Specify a binary that will be copied to the
container together with all its dependent
libraries""")
parser.add_argument("tag",
help="Image Tag")
parser.add_argument("dockerfile",
@ -164,10 +229,80 @@ class BuildCommand(SubCommand):
if dkr.image_matches_dockerfile(tag, dockerfile):
if not args.quiet:
print "Image is up to date."
else:
# Create a docker context directory for the build
docker_dir = tempfile.mkdtemp(prefix="docker_build")
# Is there a .pre file to run in the build context?
docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
if os.path.exists(docker_pre):
rc = subprocess.call(os.path.realpath(docker_pre),
cwd=docker_dir)
if rc == 3:
print "Skip"
return 0
elif rc != 0:
print "%s exited with code %d" % (docker_pre, rc)
return 1
# Do we include a extra binary?
if args.include_executable:
_copy_binary_with_libs(args.include_executable,
docker_dir)
dkr.build_image(tag, docker_dir, dockerfile,
quiet=args.quiet, argv=argv)
rmtree(docker_dir)
return 0
dkr.build_image(tag, dockerfile, args.dockerfile,
quiet=args.quiet, argv=argv)
class UpdateCommand(SubCommand):
""" Update a docker image with new executables. Arguments: <tag> <executable>"""
name = "update"
def args(self, parser):
parser.add_argument("tag",
help="Image Tag")
parser.add_argument("executable",
help="Executable to copy")
def run(self, args, argv):
# Create a temporary tarball with our whole build context and
# dockerfile for the update
tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
tmp_tar = TarFile(fileobj=tmp, mode='w')
# Add the executable to the tarball
bn = os.path.basename(args.executable)
ff = "/usr/bin/%s" % bn
tmp_tar.add(args.executable, arcname=ff)
# Add any associated libraries
libs = _get_so_libs(args.executable)
if libs:
for l in libs:
tmp_tar.add(os.path.realpath(l), arcname=l)
# Create a Docker buildfile
df = StringIO()
df.write("FROM %s\n" % args.tag)
df.write("ADD . /\n")
df.seek(0)
df_tar = TarInfo(name="Dockerfile")
df_tar.size = len(df.buf)
tmp_tar.addfile(df_tar, fileobj=df)
tmp_tar.close()
# reset the file pointers
tmp.flush()
tmp.seek(0)
# Run the build with our tarball context
dkr = Docker()
dkr.update_image(args.tag, tmp, quiet=args.quiet)
return 0
class CleanCommand(SubCommand):
@ -177,6 +312,12 @@ class CleanCommand(SubCommand):
Docker().clean()
return 0
class ImagesCommand(SubCommand):
"""Run "docker images" command"""
name = "images"
def run(self, args, argv):
return Docker().command("images", argv, args.quiet)
def main():
parser = argparse.ArgumentParser(description="A Docker helper",
usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))

21
tests/docker/dockerfiles/debian-bootstrap.docker Normal file
View File

@ -0,0 +1,21 @@
# Create Debian Bootstrap Image
#
# This is intended to be pre-poluated by:
# - a first stage debootstrap (see debian-bootstrap.pre)
# - a native qemu-$arch that binfmt_misc will run
FROM scratch
# Add everything from the context into the container
ADD . /
# Patch all mounts as docker already has stuff set up
RUN sed -i 's/in_target mount/echo not for docker in_target mount/g' /debootstrap/functions
# Run stage 2
RUN /debootstrap/debootstrap --second-stage
# At this point we can install additional packages if we want
# Duplicate deb line as deb-src
RUN cat /etc/apt/sources.list | sed "s/deb/deb-src/" >> /etc/apt/sources.list
RUN apt-get update
RUN apt-get -y build-dep qemu

87
tests/docker/dockerfiles/debian-bootstrap.pre Executable file
View File

@ -0,0 +1,87 @@
#!/bin/sh
#
# Simple wrapper for debootstrap, run in the docker build context
#
FAKEROOT=`which fakeroot 2> /dev/null`
exit_and_skip()
{
exit 3
}
#
# fakeroot is needed to run the bootstrap stage
#
if [ -z $FAKEROOT ]; then
echo "Please install fakeroot to enable bootstraping"
exit_and_skip
fi
# We check in order for
#
# - DEBOOTSTRAP_DIR pointing at a development checkout
# - PATH for the debootstrap script (installed)
#
# If neither option works then we checkout debootstrap from its
# upstream SCM and run it from there.
#
if [ -z $DEBOOTSTRAP_DIR ]; then
DEBOOTSTRAP=`which debootstrap 2> /dev/null`
if [ -z $DEBOOTSTRAP ]; then
echo "No debootstrap installed, attempting to install from SCM"
DEBOOTSTRAP_SOURCE=https://anonscm.debian.org/git/d-i/debootstrap.git
git clone ${DEBOOTSTRAP_SOURCE} ./debootstrap.git
export DEBOOTSTRAP_DIR=./debootstrap.git
DEBOOTSTRAP=./debootstrap.git/debootstrap
fi
else
DEBOOTSTRAP=${DEBOOTSTRAP_DIR}/debootstrap
if [ ! -f $DEBOOTSTRAP ]; then
echo "Couldn't find script at ${DEBOOTSTRAP}"
exit_and_skip
fi
fi
#
# Finally check to see if any qemu's are installed
#
BINFMT_DIR=/proc/sys/fs/binfmt_misc
if [ ! -e $BINFMT_DIR ]; then
echo "binfmt_misc needs enabling for a QEMU bootstrap to work"
exit_and_skip
else
# DEB_ARCH and QEMU arch names are not totally aligned
case "${DEB_ARCH}" in
amd64)
QEMU=qemu-i386
;;
armel|armhf)
QEMU=qemu-arm
;;
arm64)
QEMU=qemu-aarch64
;;
powerpc)
QEMU=qemu-ppc
;;
ppc64el)
QEMU=qemu-ppc64le
;;
s390)
QEMU=qemu-s390x
;;
*)
QEMU=qemu-${DEB_ARCH}
;;
esac
if [ ! -e "${BINFMT_DIR}/$QEMU" ]; then
echo "No binfmt_misc rule to run $QEMU, can't bootstrap"
exit_and_skip
fi
fi
echo "Building a rootfs using ${FAKEROOT} and ${DEBOOTSTRAP} ${DEB_ARCH}/${DEB_TYPE}"
${FAKEROOT} ${DEBOOTSTRAP} --variant=buildd --foreign --arch=$DEB_ARCH $DEB_TYPE . http://httpredir.debian.org/debian || exit 1
exit 0

16
tests/docker/run
View File

@ -11,6 +11,14 @@
# or (at your option) any later version. See the COPYING file in
# the top-level directory.
set -e
if test -n "$V"; then
set -x
fi
BASE="$(dirname $(readlink -e $0))"
# Prepare the environment
. /etc/profile || true
export PATH=/usr/lib/ccache:$PATH
@ -24,10 +32,10 @@ export TEST_DIR=/tmp/qemu-test
mkdir -p $TEST_DIR/{src,build,install}
# Extract the source tarballs
tar -C $TEST_DIR/src -xzf qemu.tgz
tar -C $TEST_DIR/src -xzf $BASE/qemu.tgz
for p in dtc pixman; do
if test -f $p.tgz; then
tar -C $TEST_DIR/src/$p -xzf $p.tgz
if test -f $BASE/$p.tgz; then
tar -C $TEST_DIR/src/$p -xzf $BASE/$p.tgz
export FEATURES="$FEATURES $p"
fi
done
@ -55,4 +63,6 @@ elif test -n "$DEBUG"; then
echo
# Force error after shell exits
$SHELL && exit 1
else
exit 1
fi