From 6532426aa056673b1a20e4c1efa3fc26f0567077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Fri, 28 Oct 2022 11:56:57 +0200 Subject: [PATCH 01/31] tests/lcitool: Rename non-Debian specific helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This helper is not Debian specific, rename it to cross_build(). Signed-off-by: Alex Bennée Acked-by: Richard Henderson Message-Id: <20220929114231.583801-10-alex.bennee@linaro.org> Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Message-Id: <20221028095659.48734-2-philmd@linaro.org> --- tests/lcitool/refresh | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh index e45c99adbe..a6a7c23c1d 100755 --- a/tests/lcitool/refresh +++ b/tests/lcitool/refresh @@ -93,7 +93,7 @@ debian11_extras = [ ] -def debian_cross_build(prefix, targets): +def cross_build(prefix, targets): conf = "ENV QEMU_CONFIGURE_OPTS --cross-prefix=%s\n" % (prefix) targets = "ENV DEF_TARGET_LIST %s\n" % (targets) return "".join([conf, targets]) @@ -121,45 +121,45 @@ try: # generate_dockerfile("debian-amd64-cross", "debian-11", cross="x86_64", - trailer=debian_cross_build("x86_64-linux-gnu-", - "x86_64-softmmu," - "x86_64-linux-user," - "i386-softmmu,i386-linux-user")) + trailer=cross_build("x86_64-linux-gnu-", + "x86_64-softmmu," + "x86_64-linux-user," + "i386-softmmu,i386-linux-user")) generate_dockerfile("debian-arm64-cross", "debian-11", cross="aarch64", - trailer=debian_cross_build("aarch64-linux-gnu-", - "aarch64-softmmu,aarch64-linux-user")) + trailer=cross_build("aarch64-linux-gnu-", + "aarch64-softmmu,aarch64-linux-user")) generate_dockerfile("debian-armel-cross", "debian-11", cross="armv6l", - trailer=debian_cross_build("arm-linux-gnueabi-", - "arm-softmmu,arm-linux-user,armeb-linux-user")) + trailer=cross_build("arm-linux-gnueabi-", + "arm-softmmu,arm-linux-user,armeb-linux-user")) generate_dockerfile("debian-armhf-cross", "debian-11", cross="armv7l", - trailer=debian_cross_build("arm-linux-gnueabihf-", - "arm-softmmu,arm-linux-user")) + trailer=cross_build("arm-linux-gnueabihf-", + "arm-softmmu,arm-linux-user")) generate_dockerfile("debian-mips64el-cross", "debian-11", cross="mips64el", - trailer=debian_cross_build("mips64el-linux-gnuabi64-", - "mips64el-softmmu,mips64el-linux-user")) + trailer=cross_build("mips64el-linux-gnuabi64-", + "mips64el-softmmu,mips64el-linux-user")) generate_dockerfile("debian-mipsel-cross", "debian-11", cross="mipsel", - trailer=debian_cross_build("mipsel-linux-gnu-", - "mipsel-softmmu,mipsel-linux-user")) + trailer=cross_build("mipsel-linux-gnu-", + "mipsel-softmmu,mipsel-linux-user")) generate_dockerfile("debian-ppc64el-cross", "debian-11", cross="ppc64le", - trailer=debian_cross_build("powerpc64le-linux-gnu-", - "ppc64-softmmu,ppc64-linux-user")) + trailer=cross_build("powerpc64le-linux-gnu-", + "ppc64-softmmu,ppc64-linux-user")) generate_dockerfile("debian-s390x-cross", "debian-11", cross="s390x", - trailer=debian_cross_build("s390x-linux-gnu-", - "s390x-softmmu,s390x-linux-user")) + trailer=cross_build("s390x-linux-gnu-", + "s390x-softmmu,s390x-linux-user")) # # Cirrus packages lists for GitLab From 9e243b7669054574e5cfbc2a7282dbb62827627b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Fri, 28 Oct 2022 11:56:59 +0200 Subject: [PATCH 02/31] tests/docker: update fedora-win[32|64]-cross with lcitool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert another two dockerfiles to lcitool and update. Signed-off-by: Alex Bennée Acked-by: Richard Henderson Message-Id: <20220929114231.583801-10-alex.bennee@linaro.org> Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Message-Id: <20221028095659.48734-4-philmd@linaro.org> --- .../dockerfiles/fedora-win32-cross.docker | 139 ++++++++++++------ .../dockerfiles/fedora-win64-cross.docker | 138 ++++++++++++----- tests/lcitool/refresh | 10 ++ 3 files changed, 207 insertions(+), 80 deletions(-) diff --git a/tests/docker/dockerfiles/fedora-win32-cross.docker b/tests/docker/dockerfiles/fedora-win32-cross.docker index aca37aabc4..75383ba185 100644 --- a/tests/docker/dockerfiles/fedora-win32-cross.docker +++ b/tests/docker/dockerfiles/fedora-win32-cross.docker @@ -1,46 +1,103 @@ +# THIS FILE WAS AUTO-GENERATED +# +# $ lcitool dockerfile --layers all --cross mingw32 fedora-35 qemu +# +# https://gitlab.com/libvirt/libvirt-ci + FROM registry.fedoraproject.org/fedora:35 -# Please keep this list sorted alphabetically -ENV PACKAGES \ - bc \ - bison \ - bzip2 \ - ccache \ - diffutils \ - findutils \ - flex \ - gcc \ - gettext \ - git \ - glib2-devel \ - hostname \ - make \ - meson \ - mingw32-bzip2 \ - mingw32-curl \ - mingw32-glib2 \ - mingw32-gmp \ - mingw32-gnutls \ - mingw32-gtk3 \ - mingw32-libffi \ - mingw32-libjpeg-turbo \ - mingw32-libpng \ - mingw32-libtasn1 \ - mingw32-libusbx \ - mingw32-nettle \ - mingw32-nsis \ - mingw32-pixman \ - mingw32-pkg-config \ - mingw32-SDL2 \ - msitools \ - perl \ - python3 \ - python3-PyYAML \ - tar \ - which +RUN dnf install -y nosync && \ + echo -e '#!/bin/sh\n\ +if test -d /usr/lib64\n\ +then\n\ + export LD_PRELOAD=/usr/lib64/nosync/nosync.so\n\ +else\n\ + export LD_PRELOAD=/usr/lib/nosync/nosync.so\n\ +fi\n\ +exec "$@"' > /usr/bin/nosync && \ + chmod +x /usr/bin/nosync && \ + nosync dnf update -y && \ + nosync dnf install -y \ + bash \ + bc \ + bison \ + bzip2 \ + ca-certificates \ + ccache \ + ctags \ + dbus-daemon \ + diffutils \ + findutils \ + flex \ + gcovr \ + genisoimage \ + git \ + glib2-devel \ + glibc-langpack-en \ + hostname \ + llvm \ + make \ + meson \ + ninja-build \ + nmap-ncat \ + openssh-clients \ + pcre-static \ + perl-base \ + python3 \ + python3-PyYAML \ + python3-numpy \ + python3-opencv \ + python3-pillow \ + python3-pip \ + python3-sphinx \ + python3-sphinx_rtd_theme \ + rpm \ + sed \ + sparse \ + spice-protocol \ + tar \ + tesseract \ + tesseract-langpack-eng \ + texinfo \ + util-linux \ + which && \ + nosync dnf autoremove -y && \ + nosync dnf clean all -y -RUN dnf install -y $PACKAGES -RUN rpm -q $PACKAGES | sort > /packages.txt +ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" +ENV LANG "en_US.UTF-8" +ENV MAKE "/usr/bin/make" +ENV NINJA "/usr/bin/ninja" +ENV PYTHON "/usr/bin/python3" -# Specify the cross prefix for this image (see tests/docker/common.rc) +RUN nosync dnf install -y \ + mingw32-SDL2 \ + mingw32-SDL2_image \ + mingw32-bzip2 \ + mingw32-curl \ + mingw32-gcc \ + mingw32-gcc-c++ \ + mingw32-gettext \ + mingw32-glib2 \ + mingw32-gnutls \ + mingw32-gtk3 \ + mingw32-libgcrypt \ + mingw32-libjpeg-turbo \ + mingw32-libpng \ + mingw32-libtasn1 \ + mingw32-nettle \ + mingw32-nsis \ + mingw32-pixman \ + mingw32-pkg-config && \ + nosync dnf clean all -y && \ + rpm -qa | sort > /packages.txt && \ + mkdir -p /usr/libexec/ccache-wrappers && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/i686-w64-mingw32-c++ && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/i686-w64-mingw32-cc && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/i686-w64-mingw32-g++ && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/i686-w64-mingw32-gcc + +ENV ABI "i686-w64-mingw32" +ENV MESON_OPTS "--cross-file=/usr/share/mingw/toolchain-mingw32.meson" ENV QEMU_CONFIGURE_OPTS --cross-prefix=i686-w64-mingw32- +ENV DEF_TARGET_LIST i386-softmmu diff --git a/tests/docker/dockerfiles/fedora-win64-cross.docker b/tests/docker/dockerfiles/fedora-win64-cross.docker index 3642766479..98c03dc13b 100644 --- a/tests/docker/dockerfiles/fedora-win64-cross.docker +++ b/tests/docker/dockerfiles/fedora-win64-cross.docker @@ -1,43 +1,103 @@ +# THIS FILE WAS AUTO-GENERATED +# +# $ lcitool dockerfile --layers all --cross mingw64 fedora-35 qemu +# +# https://gitlab.com/libvirt/libvirt-ci + FROM registry.fedoraproject.org/fedora:35 -# Please keep this list sorted alphabetically -ENV PACKAGES \ - bc \ - bison \ - bzip2 \ - ccache \ - diffutils \ - findutils \ - flex \ - gcc \ - gettext \ - git \ - glib2-devel \ - hostname \ - make \ - meson \ - mingw32-nsis \ - mingw64-bzip2 \ - mingw64-curl \ - mingw64-glib2 \ - mingw64-gmp \ - mingw64-gtk3 \ - mingw64-libffi \ - mingw64-libjpeg-turbo \ - mingw64-libpng \ - mingw64-libtasn1 \ - mingw64-libusbx \ - mingw64-pixman \ - mingw64-pkg-config \ - msitools \ - perl \ - python3 \ - python3-PyYAML \ - tar \ - which +RUN dnf install -y nosync && \ + echo -e '#!/bin/sh\n\ +if test -d /usr/lib64\n\ +then\n\ + export LD_PRELOAD=/usr/lib64/nosync/nosync.so\n\ +else\n\ + export LD_PRELOAD=/usr/lib/nosync/nosync.so\n\ +fi\n\ +exec "$@"' > /usr/bin/nosync && \ + chmod +x /usr/bin/nosync && \ + nosync dnf update -y && \ + nosync dnf install -y \ + bash \ + bc \ + bison \ + bzip2 \ + ca-certificates \ + ccache \ + ctags \ + dbus-daemon \ + diffutils \ + findutils \ + flex \ + gcovr \ + genisoimage \ + git \ + glib2-devel \ + glibc-langpack-en \ + hostname \ + llvm \ + make \ + meson \ + ninja-build \ + nmap-ncat \ + openssh-clients \ + pcre-static \ + perl-base \ + python3 \ + python3-PyYAML \ + python3-numpy \ + python3-opencv \ + python3-pillow \ + python3-pip \ + python3-sphinx \ + python3-sphinx_rtd_theme \ + rpm \ + sed \ + sparse \ + spice-protocol \ + tar \ + tesseract \ + tesseract-langpack-eng \ + texinfo \ + util-linux \ + which && \ + nosync dnf autoremove -y && \ + nosync dnf clean all -y -RUN dnf install -y $PACKAGES -RUN rpm -q $PACKAGES | sort > /packages.txt +ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" +ENV LANG "en_US.UTF-8" +ENV MAKE "/usr/bin/make" +ENV NINJA "/usr/bin/ninja" +ENV PYTHON "/usr/bin/python3" -# Specify the cross prefix for this image (see tests/docker/common.rc) -ENV QEMU_CONFIGURE_OPTS --cross-prefix=x86_64-w64-mingw32- --disable-capstone +RUN nosync dnf install -y \ + mingw32-nsis \ + mingw64-SDL2 \ + mingw64-SDL2_image \ + mingw64-bzip2 \ + mingw64-curl \ + mingw64-gcc \ + mingw64-gcc-c++ \ + mingw64-gettext \ + mingw64-glib2 \ + mingw64-gnutls \ + mingw64-gtk3 \ + mingw64-libgcrypt \ + mingw64-libjpeg-turbo \ + mingw64-libpng \ + mingw64-libtasn1 \ + mingw64-nettle \ + mingw64-pixman \ + mingw64-pkg-config && \ + nosync dnf clean all -y && \ + rpm -qa | sort > /packages.txt && \ + mkdir -p /usr/libexec/ccache-wrappers && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/x86_64-w64-mingw32-c++ && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/x86_64-w64-mingw32-cc && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/x86_64-w64-mingw32-g++ && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/x86_64-w64-mingw32-gcc + +ENV ABI "x86_64-w64-mingw32" +ENV MESON_OPTS "--cross-file=/usr/share/mingw/toolchain-mingw64.meson" +ENV QEMU_CONFIGURE_OPTS --cross-prefix=x86_64-w64-mingw32- +ENV DEF_TARGET_LIST x86_64-softmmu diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh index a6a7c23c1d..ce0b24c0b1 100755 --- a/tests/lcitool/refresh +++ b/tests/lcitool/refresh @@ -161,6 +161,16 @@ try: trailer=cross_build("s390x-linux-gnu-", "s390x-softmmu,s390x-linux-user")) + generate_dockerfile("fedora-win32-cross", "fedora-35", + cross="mingw32", + trailer=cross_build("i686-w64-mingw32-", + "i386-softmmu")) + + generate_dockerfile("fedora-win64-cross", "fedora-35", + cross="mingw64", + trailer=cross_build("x86_64-w64-mingw32-", + "x86_64-softmmu")) + # # Cirrus packages lists for GitLab # From 5b1229fa2dfde161ee322faaaeb35a99765c8712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Fri, 28 Oct 2022 11:56:58 +0200 Subject: [PATCH 03/31] tests/lcitool: Refresh to latest libvirt-ci module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need an updated lcitool for this to deal with the weirdness of a 32bit nsis tool for both 32 and 64 bit builds. Acked-by: Richard Henderson Message-Id: <20220929114231.583801-10-alex.bennee@linaro.org> Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Philippe Mathieu-Daudé Tested-by: Philippe Mathieu-Daudé Message-Id: <20221028095659.48734-3-philmd@linaro.org> [AJB: no longer triggers whitespace changes due to rebase] Signed-off-by: Alex Bennée --- tests/lcitool/libvirt-ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lcitool/libvirt-ci b/tests/lcitool/libvirt-ci index 79691a50a5..d40e203631 160000 --- a/tests/lcitool/libvirt-ci +++ b/tests/lcitool/libvirt-ci @@ -1 +1 @@ -Subproject commit 79691a50a5f99bd7adda236f66c3c09371b01afa +Subproject commit d40e203631eb3eacee17e8cf8fd20aa5152db62a From b1314192524a8a43f3d4fb1cae8c447a2b255308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:08 +0100 Subject: [PATCH 04/31] tests/docker: update test-mingw to run single build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While the test-build test happily run for mingw the test-mingw case runs more of the packaging inline with what our CI does. It however fails if we don't find both compilers and expects to be run on a docker image with both. Remove that distinction and make it work more like the other build test scripts. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-3-alex.bennee@linaro.org> --- tests/docker/test-mingw | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/docker/test-mingw b/tests/docker/test-mingw index 0bc6d78872..18366972eb 100755 --- a/tests/docker/test-mingw +++ b/tests/docker/test-mingw @@ -13,14 +13,12 @@ . common.rc -requires_binary x86_64-w64-mingw32-gcc -requires_binary i686-w64-mingw32-gcc +requires_binary x86_64-w64-mingw32-gcc i686-w64-mingw32-gcc cd "$BUILD_DIR" -for prefix in x86_64-w64-mingw32- i686-w64-mingw32-; do - TARGET_LIST=${TARGET_LIST:-$DEF_TARGET_LIST} \ - build_qemu --cross-prefix=$prefix \ +TARGET_LIST=${TARGET_LIST:-$DEF_TARGET_LIST} \ +build_qemu \ --enable-trace-backends=simple \ --enable-gnutls \ --enable-nettle \ @@ -29,8 +27,6 @@ for prefix in x86_64-w64-mingw32- i686-w64-mingw32-; do --enable-bzip2 \ --enable-guest-agent \ --enable-docs - install_qemu - make installer - make clean - -done +install_qemu +make installer +make clean From ed77c37ac8500d750d5858d02dc78b761dc998be Mon Sep 17 00:00:00 2001 From: Anton Johansson Date: Thu, 27 Oct 2022 19:36:09 +0100 Subject: [PATCH 05/31] tests/docker: Add flex/bison to `debian-all-test` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds flex/bison to the debian-all-test-cross container which was missed in the previous CI patch. These dependencies are required by the idef-parser patchset for target/hexagon. Signed-off-by: Anton Johansson Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20221011173229.57909-1-anjo@rev.ng> Message-Id: <20221027183637.2772968-4-alex.bennee@linaro.org> --- tests/docker/dockerfiles/debian-all-test-cross.docker | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/docker/dockerfiles/debian-all-test-cross.docker b/tests/docker/dockerfiles/debian-all-test-cross.docker index 2beb077fb4..8dc5e1b5de 100644 --- a/tests/docker/dockerfiles/debian-all-test-cross.docker +++ b/tests/docker/dockerfiles/debian-all-test-cross.docker @@ -20,8 +20,10 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ # Add extra build tools and as many cross compilers as we can for testing RUN DEBIAN_FRONTEND=noninteractive eatmydata \ apt install -y --no-install-recommends \ + bison \ ccache \ clang \ + flex \ git \ ninja-build \ gcc-aarch64-linux-gnu \ From c3b570b5a9a24d25ab522def7c928c6a3be3d842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:10 +0100 Subject: [PATCH 06/31] configure: don't enable cross compilers unless in target_list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This avoids the unfortunate effect of always builds the pc-bios blobs for targets the user isn't interested in. Reviewed-by: Richard Henderson Suggested-by: Paolo Bonzini Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-5-alex.bennee@linaro.org> --- configure | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configure b/configure index 81561be7c1..dd6f58dcde 100755 --- a/configure +++ b/configure @@ -1877,6 +1877,15 @@ probe_target_compiler() { container_cross_ranlib= container_cross_strip= + # We shall skip configuring the target compiler if the user didn't + # bother enabling an appropriate guest. This avoids building + # extraneous firmware images and tests. + if test "${target_list#*$1}" != "$1"; then + break; + else + return 1 + fi + target_arch=${1%%-*} case $target_arch in aarch64) container_hosts="x86_64 aarch64" ;; From 977cccb8451ecf0fd60388aec95dc9c1ab35afc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:11 +0100 Subject: [PATCH 07/31] configure: fix the --enable-static --disable-pie case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous tweak was incomplete as it missed a leg. Fixes: abafb64b6d (configure: explicitly set cflags for --disable-pie) Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-6-alex.bennee@linaro.org> --- configure | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/configure b/configure index dd6f58dcde..8c2c4c1a04 100755 --- a/configure +++ b/configure @@ -1327,6 +1327,8 @@ static THREAD int tls_var; int main(void) { return tls_var; } EOF +# Meson currently only handles pie as a boolean for now so if we have +# explicitly disabled PIE we need to extend our cflags because it wont. if test "$static" = "yes"; then if test "$pie" != "no" && compile_prog "-Werror -fPIE -DPIE" "-static-pie"; then CONFIGURE_CFLAGS="-fPIE -DPIE $CONFIGURE_CFLAGS" @@ -1335,13 +1337,12 @@ if test "$static" = "yes"; then error_exit "-static-pie not available due to missing toolchain support" else pie="no" + QEMU_CFLAGS="-fno-pie -no-pie $QEMU_CFLAGS" fi elif test "$pie" = "no"; then if compile_prog "-Werror -fno-pie" "-no-pie"; then CONFIGURE_CFLAGS="-fno-pie $CONFIGURE_CFLAGS" CONFIGURE_LDFLAGS="-no-pie $CONFIGURE_LDFLAGS" - # Meson currently only handles pie as a boolean for now so if we have - # explicitly disabled PIE we need to extend our cflags because it wont. QEMU_CFLAGS="-fno-pie -no-pie $QEMU_CFLAGS" fi elif compile_prog "-Werror -fPIE -DPIE" "-pie"; then From e6025635db6a6fb406a6910fc1b45ea597ea9f9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:12 +0100 Subject: [PATCH 08/31] tests/avocado: extend the timeout for x86_64 tcg tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These are timing out on gitlab. Acked-by: Richard Henderson Reviewed-by: Thomas Huth Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-7-alex.bennee@linaro.org> --- tests/avocado/boot_linux.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/avocado/boot_linux.py b/tests/avocado/boot_linux.py index b7522ad3a1..571d33882a 100644 --- a/tests/avocado/boot_linux.py +++ b/tests/avocado/boot_linux.py @@ -19,6 +19,7 @@ class BootLinuxX8664(LinuxTest): """ :avocado: tags=arch:x86_64 """ + timeout = 480 def test_pc_i440fx_tcg(self): """ From b4c82b1b4d030b0b3edbab7173b6888aa12e7e0a Mon Sep 17 00:00:00 2001 From: Anton Johansson Date: Thu, 27 Oct 2022 19:36:13 +0100 Subject: [PATCH 09/31] tests/docker: Add flex/bison to `debian-hexagon-cross` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit debian-hexagon-cross contains two images, one to build the toolchain used for building the Hexagon tests themselves, and one image to build QEMU and run the tests. This commit adds flex/bison to the final image that builds QEMU so that it can also build idef-parser. Note: This container is not built by the CI and needs to be rebuilt and updated manually. Signed-off-by: Anton Johansson Reviewed-by: Thomas Huth Message-Id: <20221014223642.147845-1-anjo@rev.ng> Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-8-alex.bennee@linaro.org> --- tests/docker/dockerfiles/debian-hexagon-cross.docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/dockerfiles/debian-hexagon-cross.docker b/tests/docker/dockerfiles/debian-hexagon-cross.docker index 8d219bb81d..c4238e893f 100644 --- a/tests/docker/dockerfiles/debian-hexagon-cross.docker +++ b/tests/docker/dockerfiles/debian-hexagon-cross.docker @@ -43,7 +43,7 @@ RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.lis # Install QEMU build deps for use in CI RUN apt update && \ DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \ - DEBIAN_FRONTEND=noninteractive eatmydata apt install -yy git ninja-build && \ + DEBIAN_FRONTEND=noninteractive eatmydata apt install -yy bison flex git ninja-build && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt build-dep -yy --arch-only qemu COPY --from=0 /usr/local /usr/local From 991e9051238e15207ca3fbfc630fd98f0e2b86cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:14 +0100 Subject: [PATCH 10/31] tests/tcg: use regular semihosting for nios2-softmmu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nios2 code now plugs into the common semihosting code so we can use the same redirect invocation as the other boards. There is however a bug raised for the fact the tests don't seem to be completing properly and silently passing anyway: https://gitlab.com/qemu-project/qemu/-/issues/1258 Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-9-alex.bennee@linaro.org> --- tests/tcg/nios2/Makefile.softmmu-target | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/tcg/nios2/Makefile.softmmu-target b/tests/tcg/nios2/Makefile.softmmu-target index c3d0594a39..bc7fd55060 100644 --- a/tests/tcg/nios2/Makefile.softmmu-target +++ b/tests/tcg/nios2/Makefile.softmmu-target @@ -25,8 +25,7 @@ LDFLAGS += -Wl,-T$(LINK_SCRIPT) -static -nostdlib $(CRT_OBJS) -lgcc %: %.o $(LINK_SCRIPT) $(CRT_OBJS) $(call quiet-command, $(CC) $(CFLAGS) $(EXTRA_CFLAGS) $< -o $@ $(LDFLAGS), LD, $@) -# FIXME: nios2 semihosting writes to stdout, not a chardev -QEMU_OPTS = -M 10m50-ghrd,vic=on -semihosting >$@.out -kernel +QEMU_OPTS = -M 10m50-ghrd,vic=on -semihosting-config enable=on,target=native,chardev=output -kernel memory: CFLAGS+=-DCHECK_UNALIGNED=0 TESTS += $(MULTIARCH_TESTS) From 25916dd65d7e72f7bb5b9727bb346a9d17993072 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 27 Oct 2022 19:36:15 +0100 Subject: [PATCH 11/31] tests/tcg/nios2: Tweak 10m50-ghrd.ld MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit More closely follow the default linker script for nios2. This magically fixes a problem resolving .got relocs from the toolchain's libgcc.a. Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1258 Signed-off-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20221024035341.2971123-1-richard.henderson@linaro.org> Message-Id: <20221027183637.2772968-10-alex.bennee@linaro.org> --- tests/tcg/nios2/10m50-ghrd.ld | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/tcg/nios2/10m50-ghrd.ld b/tests/tcg/nios2/10m50-ghrd.ld index 7db0d59ad7..71cdda450c 100644 --- a/tests/tcg/nios2/10m50-ghrd.ld +++ b/tests/tcg/nios2/10m50-ghrd.ld @@ -44,11 +44,15 @@ SECTIONS .data : ALIGN(4) { *(.shdata) *(.data .data.* .gnu.linkonce.d.*) - . = ALIGN(4); - _gp = ABSOLUTE(. + 0x8000); - *(.got.plt) *(.got) - *(.lit8) - *(.lit4) + } >ram :RAM + + HIDDEN (_gp = ALIGN(16) + 0x7ff0); + PROVIDE_HIDDEN (gp = _gp); + .got : ALIGN(4) { + *(.got.plt) *(.igot.plt) *(.got) *(.igot) + } >ram :RAM + + .sdata : ALIGN(4) { *(.sdata .sdata.* .gnu.linkonce.s.*) } >ram :RAM From 3878d0c7d7de9fe201513b8ee31e38e53361a97d Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Thu, 27 Oct 2022 19:36:16 +0100 Subject: [PATCH 12/31] semihosting/arm-compat-semi: Avoid using hardcoded /tmp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use g_get_tmp_dir() to get the directory to use for temporary files. Signed-off-by: Bin Meng Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20221006151927.2079583-2-bmeng.cn@gmail.com> Message-Id: <20221027183637.2772968-11-alex.bennee@linaro.org> --- semihosting/arm-compat-semi.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/semihosting/arm-compat-semi.c b/semihosting/arm-compat-semi.c index bfea9e9337..62d8bae97f 100644 --- a/semihosting/arm-compat-semi.c +++ b/semihosting/arm-compat-semi.c @@ -503,7 +503,8 @@ void do_common_semihosting(CPUState *cs) GET_ARG(0); GET_ARG(1); GET_ARG(2); - len = asprintf(&s, "/tmp/qemu-%x%02x", getpid(), (int)arg1 & 0xff); + len = asprintf(&s, "%s/qemu-%x%02x", g_get_tmp_dir(), + getpid(), (int)arg1 & 0xff); if (len < 0) { common_semi_set_ret(cs, -1); break; From eb6b2edf8e1a5845c81f3c4da905f1f8bbc0dec3 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Thu, 27 Oct 2022 19:36:17 +0100 Subject: [PATCH 13/31] tcg: Avoid using hardcoded /tmp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use g_get_tmp_dir() to get the directory to use for temporary files. Signed-off-by: Bin Meng Reviewed-by: Marc-André Lureau Reviewed-by: Alex Bennée Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20221006151927.2079583-3-bmeng.cn@gmail.com> Message-Id: <20221027183637.2772968-12-alex.bennee@linaro.org> --- tcg/tcg.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 612a12f58f..84921b64f7 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -4729,7 +4729,8 @@ static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, /* Enable this block to be able to debug the ELF image file creation. One can use readelf, objdump, or other inspection utilities. */ { - FILE *f = fopen("/tmp/qemu.jit", "w+b"); + g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir()); + FILE *f = fopen(jit, "w+b"); if (f) { if (fwrite(img, img_size, 1, f) != img_size) { /* Avoid stupid unused return value warning for fwrite. */ From c2632994e93eee2f165113f1c78775b113f99926 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Thu, 27 Oct 2022 19:36:19 +0100 Subject: [PATCH 14/31] block/vvfat: Unify the mkdir() call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is a difference in the mkdir() call for win32 and non-win32 platforms, and currently is handled in the codes with #ifdefs. glib provides a portable g_mkdir() API and we can use it to unify the codes without #ifdefs. Signed-off-by: Bin Meng Reviewed-by: Marc-André Lureau Signed-off-by: Alex Bennée Reviewed-by: Kevin Wolf Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20221006151927.2079583-6-bmeng.cn@gmail.com> Message-Id: <20221027183637.2772968-14-alex.bennee@linaro.org> --- block/vvfat.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/block/vvfat.c b/block/vvfat.c index c5b1442145..723c91216e 100644 --- a/block/vvfat.c +++ b/block/vvfat.c @@ -25,6 +25,7 @@ #include "qemu/osdep.h" #include +#include #include "qapi/error.h" #include "block/block_int.h" #include "block/qdict.h" @@ -2726,13 +2727,9 @@ static int handle_renames_and_mkdirs(BDRVVVFATState* s) mapping_t* mapping; int j, parent_path_len; -#ifdef __MINGW32__ - if (mkdir(commit->path)) + if (g_mkdir(commit->path, 0755)) { return -5; -#else - if (mkdir(commit->path, 0755)) - return -5; -#endif + } mapping = insert_mapping(s, commit->param.mkdir.cluster, commit->param.mkdir.cluster + 1); From 34b55848a15bca120d9b9381881c40b045409ee9 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Thu, 27 Oct 2022 19:36:20 +0100 Subject: [PATCH 15/31] hw/usb: dev-mtp: Use g_mkdir() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use g_mkdir() to create a directory on all platforms. Signed-off-by: Bin Meng Acked-by: Gerd Hoffmann Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20221006151927.2079583-8-bmeng.cn@gmail.com> Message-Id: <20221027183637.2772968-15-alex.bennee@linaro.org> --- hw/usb/dev-mtp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/usb/dev-mtp.c b/hw/usb/dev-mtp.c index 5831395cef..1cac1cd435 100644 --- a/hw/usb/dev-mtp.c +++ b/hw/usb/dev-mtp.c @@ -14,7 +14,7 @@ #include "qemu/error-report.h" #include #include - +#include #include @@ -1622,7 +1622,7 @@ static void usb_mtp_write_data(MTPState *s, uint32_t handle) if (s->dataset.filename) { path = g_strdup_printf("%s/%s", parent->path, s->dataset.filename); if (s->dataset.format == FMT_ASSOCIATION) { - ret = mkdir(path, mask); + ret = g_mkdir(path, mask); if (!ret) { usb_mtp_queue_result(s, RES_OK, d->trans, 3, QEMU_STORAGE_ID, From 48fad83ff49bd47368223cf1121351f51cf3565f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:21 +0100 Subject: [PATCH 16/31] MAINTAINERS: add entries for the key build bits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes to the build files are a bit special in that they usually go through other maintainer trees. However considering the build system is the root of everything a developer is likely to do we should at least set it out in MAINTAINERS. I'm going to nominate Paolo for meson stuff given the conversion was his passion project. I'm happy to cast an eye over configure stuff considering a lot of the cross compile logic is in there anyway. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Acked-by: Thomas Huth Cc: Paolo Bonzini Message-Id: <20221027183637.2772968-16-alex.bennee@linaro.org> --- MAINTAINERS | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index c41d8d65e2..663f1b6581 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -547,12 +547,14 @@ F: */*win32* F: include/*/*win32* X: qga/*win32* F: qemu.nsi +F: scripts/nsis.py Darwin (macOS, iOS) M: Philippe Mathieu-Daudé S: Odd Fixes F: .gitlab-ci.d/cirrus/macos-* F: */*.m +F: scripts/entitlement.sh Alpha Machines -------------- @@ -3762,6 +3764,29 @@ F: docs/about/deprecated.rst Build System ------------ +Meson +M: Paolo Bonzini +R: Marc-André Lureau +R: Daniel P. Berrange +R: Thomas Huth +R: Philippe Mathieu-Daudé +S: Maintained +F: meson.build +F: meson_options.txt +F: scripts/meson-buildoptions.* +F: scripts/check_sparse.py +F: scripts/symlink-install-tree.py + +Top Level Makefile and configure +M: Paolo Bonzini +R: Alex Bennée +R: Thomas Huth +S: Maintained +F: Makefile +F: configure +F: scripts/mtest2make.py +F: tests/Makefile.include + GIT submodules M: Daniel P. Berrange S: Odd Fixes From 97cfba13996067a572db168214e3ccbc22f7adaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:22 +0100 Subject: [PATCH 17/31] MAINTAINERS: add features_to_c.sh to gdbstub files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-17-alex.bennee@linaro.org> --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 663f1b6581..8a466fd6f4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2687,6 +2687,7 @@ F: gdbstub/* F: include/exec/gdbstub.h F: gdb-xml/ F: tests/tcg/multiarch/gdbstub/ +F: scripts/feature_to_c.sh Memory API M: Paolo Bonzini From 162f916453fb9e2d1d1c45cd081a8f8c66f29b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:23 +0100 Subject: [PATCH 18/31] MAINTAINERS: fix-up for check-tcg Makefile changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: fc76c56d3f ("tests/tcg: cleanup Makefile inclusions") Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-18-alex.bennee@linaro.org> --- MAINTAINERS | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 8a466fd6f4..f59b568e5b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3727,8 +3727,7 @@ Guest Test Compilation Support M: Alex Bennée R: Philippe Mathieu-Daudé S: Maintained -F: tests/tcg/Makefile -F: tests/tcg/Makefile.include +F: tests/tcg/Makefile.target Integration Testing with the Avocado framework W: https://trello.com/b/6Qi1pxVn/avocado-qemu From 5104b73824670316b1ae1736054e4ba8ab77f345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:24 +0100 Subject: [PATCH 19/31] tests/avocado: set -machine none for userfwd and vnc tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These are exercising core QEMU features and don't actually run code. Not specifying a machine will fail when avocado chooses the native arch binary to run. Be explicit. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-19-alex.bennee@linaro.org> --- tests/avocado/info_usernet.py | 3 +++ tests/avocado/vnc.py | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/avocado/info_usernet.py b/tests/avocado/info_usernet.py index b862a47dba..fdc4d90c42 100644 --- a/tests/avocado/info_usernet.py +++ b/tests/avocado/info_usernet.py @@ -14,6 +14,9 @@ from qemu.utils import get_info_usernet_hostfwd_port class InfoUsernet(QemuSystemTest): + """ + :avocado: tags=machine:none + """ def test_hostfwd(self): self.require_netdev('user') diff --git a/tests/avocado/vnc.py b/tests/avocado/vnc.py index 187fd3febc..aeeefc70be 100644 --- a/tests/avocado/vnc.py +++ b/tests/avocado/vnc.py @@ -53,6 +53,7 @@ def find_free_ports(count: int) -> List[int]: class Vnc(QemuSystemTest): """ :avocado: tags=vnc,quick + :avocado: tags=machine:none """ def test_no_vnc(self): self.vm.add_args('-nodefaults', '-S') From cc45d25c65cf69b54fda0794ef74f53daaa272c2 Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Thu, 27 Oct 2022 19:36:25 +0100 Subject: [PATCH 20/31] tests/avocado: raspi2_initrd: Wait for guest shutdown message before stopping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The avocado test tests/avocado/boot_linux_console.py:BootLinuxConsole.test_arm_raspi2_initrd finishes with exec_command(self, 'halt') # Wait for VM to shut down gracefully self.vm.wait() In theory this should be fine. In practice it runs into two bugs: * when the test calls self.vm.wait() Avocado closes the socket connection to the guest serial console immediately, so the avocado logs don't have the last part of the guest output: https://gitlab.com/qemu-project/qemu/-/issues/1265 * when the socket is closed, a bug in the QEMU socket chardev means that it loses any data that the guest UART has not yet consumed. This means that the guest doesn't always read the full 'halt' command string, so the test intermittently fails with a timeout: https://gitlab.com/qemu-project/qemu/-/issues/1264 Work around both of these by waiting for the guest to print the string that means it has completed the shutdown process. This fixes a very long standing intermittent failure in this test. Resolves: https://gitlab.com/qemu-project/qemu/-/issues/636 Signed-off-by: Peter Maydell Signed-off-by: Alex Bennée Reviewed-by: Richard Henderson Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Thomas Huth Message-Id: <20221020102012.3015662-1-peter.maydell@linaro.org> Message-Id: <20221027183637.2772968-20-alex.bennee@linaro.org> --- tests/avocado/boot_linux_console.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/avocado/boot_linux_console.py b/tests/avocado/boot_linux_console.py index ca9d09b0d7..eed4b49e6e 100644 --- a/tests/avocado/boot_linux_console.py +++ b/tests/avocado/boot_linux_console.py @@ -489,7 +489,7 @@ class BootLinuxConsole(LinuxKernelTest): 'BCM2835') exec_command_and_wait_for_pattern(self, 'cat /proc/iomem', '/soc/cprman@7e101000') - exec_command(self, 'halt') + exec_command_and_wait_for_pattern(self, 'halt', 'reboot: System halted') # Wait for VM to shut down gracefully self.vm.wait() From 500f73b1aa24b10278a6bf496ad3263b24964388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:26 +0100 Subject: [PATCH 21/31] tests/avocado: disable sh4 rd2 tests on Gitlab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running repeated invocations on a number of test boxes show a fairly high error rate: $ retry.py -n 100 -c -- ./tests/venv/bin/avocado run tests/avocado/boot_linux_console.py:BootLinuxConsole.test_sh4_r2d retry.py called with ['./tests/venv/bin/avocado', 'run', 'tests/avocado/boot_linux_console.py:BootLinuxConsole.test_sh4_r2d'] Results: Run, Ret, Pass/Fail, Time, Total Pass, Total Run ... Results summary: 0: 94 times (94.00%), avg time 2.254 (0.00 varience/0.04 deviation) 1: 3 times (3.00%), avg time 1.837 (0.02 varience/0.14 deviation) 8: 3 times (3.00%), avg time 91.288 (0.02 varience/0.15 deviation) Examining the logs they fall into various categories of un-handled unaligned access by user space and unexpected FPU usage by the kernel which ultimately lead to the failure to reach the login prompt. This could be bugs in the translator that only get hit occasionally or just a flaky kernel - its hard to tell. To avoid these failures gating CI lets skip on GitLab. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Cc: Thomas Huth Cc: Yoshinori Sato Cc: Magnus Damm Message-Id: <20221027183637.2772968-21-alex.bennee@linaro.org> --- tests/avocado/boot_linux_console.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/avocado/boot_linux_console.py b/tests/avocado/boot_linux_console.py index eed4b49e6e..4c9d551f47 100644 --- a/tests/avocado/boot_linux_console.py +++ b/tests/avocado/boot_linux_console.py @@ -15,6 +15,7 @@ import shutil from avocado import skip from avocado import skipUnless +from avocado import skipIf from avocado_qemu import QemuSystemTest from avocado_qemu import exec_command from avocado_qemu import exec_command_and_wait_for_pattern @@ -1175,6 +1176,10 @@ class BootLinuxConsole(LinuxKernelTest): self.vm.add_args('-M', 'graphics=off') self.do_test_advcal_2018('15', tar_hash, 'invaders.elf') + # This test has a 6-10% failure rate on various hosts that look + # like issues with a buggy kernel. As a result we don't want it + # gating releases on Gitlab. + @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_sh4_r2d(self): """ :avocado: tags=arch:sh4 From bc483a91d5a2182387e82913a7e22c86b6b1c85c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:27 +0100 Subject: [PATCH 22/31] tests/tcg: re-enable linux-test for sh4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test was marked as broken due to bug #704 which was fixed by aee14c77f4 (linux-user: Rewrite do_getdents, do_getdents64). Local testing shows this is solid now so lets re-enable the test. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Cc: Yoshinori Sato Message-Id: <20221027183637.2772968-22-alex.bennee@linaro.org> --- tests/tcg/sh4/Makefile.target | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/tcg/sh4/Makefile.target b/tests/tcg/sh4/Makefile.target index 35ebe6b4e3..32b019bdf1 100644 --- a/tests/tcg/sh4/Makefile.target +++ b/tests/tcg/sh4/Makefile.target @@ -13,12 +13,6 @@ run-signals: signals run-plugin-signals-with-%: $(call skip-test, $<, "BROKEN") -# This test is currently broken: https://gitlab.com/qemu-project/qemu/-/issues/704 -run-linux-test: linux-test - $(call skip-test, $<, "BROKEN") -run-plugin-linux-test-with-%: - $(call skip-test, $<, "BROKEN") - # This test is currently unreliable: https://gitlab.com/qemu-project/qemu/-/issues/856 run-threadcount: $(call skip-test, $<, "BROKEN") From 839866fcbd4f2ce554f3c16bf68a520ebc2da310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:28 +0100 Subject: [PATCH 23/31] tests/tcg: re-enable threadcount for sh4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test was marked as broken due to bug #856 which was fixed by ab419fd8a0 (target/sh4: Fix TB_FLAG_UNALIGN). Local testing shows this is solid now so lets re-enable the test. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Alex Bennée Cc: Yoshinori Sato Message-Id: <20221027183637.2772968-23-alex.bennee@linaro.org> --- tests/tcg/sh4/Makefile.target | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/tcg/sh4/Makefile.target b/tests/tcg/sh4/Makefile.target index 32b019bdf1..47c39a44b6 100644 --- a/tests/tcg/sh4/Makefile.target +++ b/tests/tcg/sh4/Makefile.target @@ -12,9 +12,3 @@ run-signals: signals $(call skip-test, $<, "BROKEN") run-plugin-signals-with-%: $(call skip-test, $<, "BROKEN") - -# This test is currently unreliable: https://gitlab.com/qemu-project/qemu/-/issues/856 -run-threadcount: - $(call skip-test, $<, "BROKEN") -run-plugin-threadcount-with-%: - $(call skip-test, $<, "BROKEN") From efe7c4f08d543cd4181c672bedba5575390db2a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:29 +0100 Subject: [PATCH 24/31] target/s390x: don't use ld_code2 to probe next pc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This isn't an translator picking up an instruction so we shouldn't use the translator_lduw function which has side effects for plugins. Reviewed-by: David Hildenbrand Reviewed-by: Richard Henderson Acked-by: Ilya Leoshkevich Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-24-alex.bennee@linaro.org> --- target/s390x/tcg/translate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index 5798928473..9df7f9e693 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -6612,7 +6612,7 @@ static void s390x_tr_insn_start(DisasContextBase *dcbase, CPUState *cs) static target_ulong get_next_pc(CPUS390XState *env, DisasContext *s, uint64_t pc) { - uint64_t insn = ld_code2(env, s, pc); + uint64_t insn = cpu_lduw_code(env, pc); return pc + get_ilen((insn >> 8) & 0xff); } From 621aab6c7dffaa37a9d9b89d246894e516eb3c46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:30 +0100 Subject: [PATCH 25/31] target/s390x: don't probe next pc for EXecuted insns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have finished the TB anyway so we can shortcut the other tests by checking dc->ex_value first. Reviewed-by: David Hildenbrand Reviewed-by: Richard Henderson Acked-by: Ilya Leoshkevich Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-25-alex.bennee@linaro.org> --- target/s390x/tcg/translate.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index 9df7f9e693..f4122db434 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -6624,9 +6624,9 @@ static void s390x_tr_translate_insn(DisasContextBase *dcbase, CPUState *cs) dc->base.is_jmp = translate_one(env, dc); if (dc->base.is_jmp == DISAS_NEXT) { - if (!is_same_page(dcbase, dc->base.pc_next) || - !is_same_page(dcbase, get_next_pc(env, dc, dc->base.pc_next)) || - dc->ex_value) { + if (dc->ex_value || + !is_same_page(dcbase, dc->base.pc_next) || + !is_same_page(dcbase, get_next_pc(env, dc, dc->base.pc_next))) { dc->base.is_jmp = DISAS_TOO_MANY; } } From 9fa97e04ae5677cc383b674f8efbf17950b07622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:31 +0100 Subject: [PATCH 26/31] target/s390x: fake instruction loading when handling 'ex' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The s390x EXecute instruction is a bit weird as we synthesis the executed instruction from what we have stored in memory. This missed the plugin instrumentation. Work around this with a special helper to inform the rest of the translator about the instruction so things stay consistent. Reviewed-by: David Hildenbrand Acked-by: Ilya Leoshkevich Reviewed-by: Richard Henderson Signed-off-by: Alex Bennée Cc: Richard Henderson Message-Id: <20221027183637.2772968-26-alex.bennee@linaro.org> --- include/exec/translator.h | 17 +++++++++++++++++ target/s390x/tcg/translate.c | 6 ++++++ 2 files changed, 23 insertions(+) diff --git a/include/exec/translator.h b/include/exec/translator.h index 3b77f5f4aa..af2ff95cd5 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -211,6 +211,23 @@ translator_ldq_swap(CPUArchState *env, DisasContextBase *db, return ret; } +/** + * translator_fake_ldb - fake instruction load + * @insn8: byte of instruction + * @pc: program counter of instruction + * + * This is a special case helper used where the instruction we are + * about to translate comes from somewhere else (e.g. being + * re-synthesised for s390x "ex"). It ensures we update other areas of + * the translator with details of the executed instruction. + */ + +static inline void translator_fake_ldb(uint8_t insn8, abi_ptr pc) +{ + plugin_insn_append(pc, &insn8, sizeof(insn8)); +} + + /* * Return whether addr is on the same page as where disassembly started. * Translators can use this to enforce the rule that only single-insn diff --git a/target/s390x/tcg/translate.c b/target/s390x/tcg/translate.c index f4122db434..03efccdf9f 100644 --- a/target/s390x/tcg/translate.c +++ b/target/s390x/tcg/translate.c @@ -6317,12 +6317,18 @@ static const DisasInsn *extract_insn(CPUS390XState *env, DisasContext *s) if (unlikely(s->ex_value)) { /* Drop the EX data now, so that it's clear on exception paths. */ TCGv_i64 zero = tcg_const_i64(0); + int i; tcg_gen_st_i64(zero, cpu_env, offsetof(CPUS390XState, ex_value)); tcg_temp_free_i64(zero); /* Extract the values saved by EXECUTE. */ insn = s->ex_value & 0xffffffffffff0000ull; ilen = s->ex_value & 0xf; + /* register insn bytes with translator so plugins work */ + for (i = 0; i < ilen; i++) { + uint8_t byte = extract64(insn, 56 - (i * 8), 8); + translator_fake_ldb(byte, pc + i); + } op = insn >> 56; } else { insn = ld_code2(env, s, pc); From d1d94d968d30ce57674b7e3f2752657e1cd2e4b6 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 27 Oct 2022 19:36:32 +0100 Subject: [PATCH 27/31] tests/tcg: include CONFIG_PLUGIN in config-host.mak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paolo Bonzini Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20221013131304.623740-1-pbonzini@redhat.com> Message-Id: <20221027183637.2772968-27-alex.bennee@linaro.org> --- configure | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure b/configure index 8c2c4c1a04..4275f5419f 100755 --- a/configure +++ b/configure @@ -2486,6 +2486,9 @@ echo "HOST_CC=$host_cc" >> $config_host_mak if test -n "$gdb_bin"; then echo "HAVE_GDB_BIN=$gdb_bin" >> $config_host_mak fi +if test "$plugins" = "yes" ; then + echo "CONFIG_PLUGIN=y" >> $config_host_mak +fi tcg_tests_targets= for target in $target_list; do From a4cc0b7dd7dc27f6afb5d4e0338d953e36319c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:33 +0100 Subject: [PATCH 28/31] contrib/plugins: enable debug on CONFIG_DEBUG_TCG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We used to rely on QEMU_CFLAGS to expose the debug flags but now this is synthesised by meson and only available to the main build. Add our own flags if we detect the build has been enabled with CONFIG_DEBUG_TCG (which is the default for --enable-debug anyway). Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Cc: Paolo Bonzini Message-Id: <20221027183637.2772968-28-alex.bennee@linaro.org> --- contrib/plugins/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/plugins/Makefile b/contrib/plugins/Makefile index df3499f4f2..23e0396687 100644 --- a/contrib/plugins/Makefile +++ b/contrib/plugins/Makefile @@ -29,6 +29,7 @@ SONAMES := $(addsuffix .so,$(addprefix lib,$(NAMES))) CFLAGS = $(GLIB_CFLAGS) CFLAGS += -fPIC -Wall $(filter -W%, $(QEMU_CFLAGS)) CFLAGS += $(if $(findstring no-psabi,$(QEMU_CFLAGS)),-Wpsabi) +CFLAGS += $(if $(CONFIG_DEBUG_TCG), -ggdb -O0) CFLAGS += -I$(SRC_PATH)/include/qemu all: $(SONAMES) From 14fd492b89b512735ac3abf8f5300db865129378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:34 +0100 Subject: [PATCH 29/31] contrib/plugins: protect execlog's last_exec expansion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We originally naively treated expansion as safe because we expected each new CPU/thread to appear in order. However the -M raspi2 model triggered a case where a new high cpu_index thread started executing just before a smaller one. Clean this up by converting the GArray into the simpler GPtrArray and then holding a lock for the expansion. Signed-off-by: Alex Bennée Cc: Alexandre Iooss Reviewed-by: Richard Henderson Message-Id: <20221027183637.2772968-29-alex.bennee@linaro.org> --- contrib/plugins/execlog.c | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/contrib/plugins/execlog.c b/contrib/plugins/execlog.c index 1b3bb7ebba..e255bd21fd 100644 --- a/contrib/plugins/execlog.c +++ b/contrib/plugins/execlog.c @@ -18,11 +18,30 @@ QEMU_PLUGIN_EXPORT int qemu_plugin_version = QEMU_PLUGIN_VERSION; /* Store last executed instruction on each vCPU as a GString */ -GArray *last_exec; +static GPtrArray *last_exec; +static GMutex expand_array_lock; static GPtrArray *imatches; static GArray *amatches; +/* + * Expand last_exec array. + * + * As we could have multiple threads trying to do this we need to + * serialise the expansion under a lock. Threads accessing already + * created entries can continue without issue even if the ptr array + * gets reallocated during resize. + */ +static void expand_last_exec(int cpu_index) +{ + g_mutex_lock(&expand_array_lock); + while (cpu_index >= last_exec->len) { + GString *s = g_string_new(NULL); + g_ptr_array_add(last_exec, s); + } + g_mutex_unlock(&expand_array_lock); +} + /** * Add memory read or write information to current instruction log */ @@ -33,7 +52,7 @@ static void vcpu_mem(unsigned int cpu_index, qemu_plugin_meminfo_t info, /* Find vCPU in array */ g_assert(cpu_index < last_exec->len); - s = g_array_index(last_exec, GString *, cpu_index); + s = g_ptr_array_index(last_exec, cpu_index); /* Indicate type of memory access */ if (qemu_plugin_mem_is_store(info)) { @@ -61,11 +80,10 @@ static void vcpu_insn_exec(unsigned int cpu_index, void *udata) GString *s; /* Find or create vCPU in array */ - while (cpu_index >= last_exec->len) { - s = g_string_new(NULL); - g_array_append_val(last_exec, s); + if (cpu_index >= last_exec->len) { + expand_last_exec(cpu_index); } - s = g_array_index(last_exec, GString *, cpu_index); + s = g_ptr_array_index(last_exec, cpu_index); /* Print previous instruction in cache */ if (s->len) { @@ -163,7 +181,7 @@ static void plugin_exit(qemu_plugin_id_t id, void *p) guint i; GString *s; for (i = 0; i < last_exec->len; i++) { - s = g_array_index(last_exec, GString *, i); + s = g_ptr_array_index(last_exec, i); if (s->str) { qemu_plugin_outs(s->str); qemu_plugin_outs("\n"); @@ -201,7 +219,11 @@ QEMU_PLUGIN_EXPORT int qemu_plugin_install(qemu_plugin_id_t id, * Initialize dynamic array to cache vCPU instruction. In user mode * we don't know the size before emulation. */ - last_exec = g_array_new(FALSE, FALSE, sizeof(GString *)); + if (info->system_emulation) { + last_exec = g_ptr_array_sized_new(info->system.max_vcpus); + } else { + last_exec = g_ptr_array_new(); + } for (int i = 0; i < argc; i++) { char *opt = argv[i]; From 68406d10859385c88da73d0106254a7f47e6652e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Thu, 27 Oct 2022 19:36:35 +0100 Subject: [PATCH 30/31] tests/unit: cleanups for test-io-channel-command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test is hanging under heavy load when the two socats race while trying to create the socket. I've tried various approaches to avoid the race but it seems "creat=0" won't stop socat trying to create a pipe if it executes first. In the end I just use a small sleep which seems to be reliable enough on the load situations I've tried. While I was there I also properly created a tmpdir for the socket to live in which is cleaned up at the end of the test. Signed-off-by: Alex Bennée Reviewed-by: Daniel P. Berrangé Cc: Thomas Huth Cc: Marc-André Lureau Cc: Juan Quintela Message-Id: <20221027183637.2772968-30-alex.bennee@linaro.org> --- tests/unit/test-io-channel-command.c | 45 +++++++++++++++++----------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/tests/unit/test-io-channel-command.c b/tests/unit/test-io-channel-command.c index 7eee939c07..43e29c8cfb 100644 --- a/tests/unit/test-io-channel-command.c +++ b/tests/unit/test-io-channel-command.c @@ -19,6 +19,7 @@ */ #include "qemu/osdep.h" +#include #include "io/channel-command.h" #include "io-channel-helpers.h" #include "qapi/error.h" @@ -26,32 +27,32 @@ #define TEST_FIFO "test-io-channel-command.fifo" -#define SOCAT_SRC "PIPE:" TEST_FIFO ",wronly" -#define SOCAT_DST "PIPE:" TEST_FIFO ",rdonly" - static char *socat = NULL; static void test_io_channel_command_fifo(bool async) { + g_autofree gchar *tmpdir = g_dir_make_tmp("qemu-test-io-channel.XXXXXX", NULL); + g_autofree gchar *fifo = g_strdup_printf("%s/%s", tmpdir, TEST_FIFO); + g_autoptr(GString) srcargs = g_string_new(socat); + g_autoptr(GString) dstargs = g_string_new(socat); + g_auto(GStrv) srcargv; + g_auto(GStrv) dstargv; QIOChannel *src, *dst; QIOChannelTest *test; - const char *srcargv[] = { - socat, "-", SOCAT_SRC, NULL, - }; - const char *dstargv[] = { - socat, SOCAT_DST, "-", NULL, - }; - if (!socat) { - g_test_skip("socat is not found in PATH"); - return; - } + g_string_append_printf(srcargs, " - PIPE:%s,wronly", fifo); + g_string_append_printf(dstargs, " PIPE:%s,rdonly -", fifo); - unlink(TEST_FIFO); - src = QIO_CHANNEL(qio_channel_command_new_spawn(srcargv, + srcargv = g_strsplit(srcargs->str, " ", -1); + dstargv = g_strsplit(dstargs->str, " ", -1); + + src = QIO_CHANNEL(qio_channel_command_new_spawn((const char **) srcargv, O_WRONLY, &error_abort)); - dst = QIO_CHANNEL(qio_channel_command_new_spawn(dstargv, + /* try to avoid a race to create the socket */ + g_usleep(1000); + + dst = QIO_CHANNEL(qio_channel_command_new_spawn((const char **) dstargv, O_RDONLY, &error_abort)); @@ -62,17 +63,27 @@ static void test_io_channel_command_fifo(bool async) object_unref(OBJECT(src)); object_unref(OBJECT(dst)); - unlink(TEST_FIFO); + g_rmdir(tmpdir); } static void test_io_channel_command_fifo_async(void) { + if (!socat) { + g_test_skip("socat is not found in PATH"); + return; + } + test_io_channel_command_fifo(true); } static void test_io_channel_command_fifo_sync(void) { + if (!socat) { + g_test_skip("socat is not found in PATH"); + return; + } + test_io_channel_command_fifo(false); } From 339bf0c071eff5e6ff1d9ddb3ad5cd02e4cd9ca3 Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Thu, 27 Oct 2022 19:36:36 +0100 Subject: [PATCH 31/31] tests/vm: use -o IdentitiesOnly=yes for ssh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When one has a lot of keys in ~/.ssh directory, the ssh command will try all of them before the one specified on the command line, and this may cause the remote ssh server to reject the connection due to too many failed authentication attempts. Fix by adding -o IdentitiesOnly=yes, which makes the ssh client consider only the keys specified on the command line. Signed-off-by: Ilya Leoshkevich Reviewed-by: Thomas Huth Message-Id: <20221027113026.2280863-1-iii@linux.ibm.com> Signed-off-by: Alex Bennée Message-Id: <20221027183637.2772968-31-alex.bennee@linaro.org> --- tests/vm/basevm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py index 4fd9af10b7..2276364c42 100644 --- a/tests/vm/basevm.py +++ b/tests/vm/basevm.py @@ -233,7 +233,8 @@ class BaseVM(object): "-o", "UserKnownHostsFile=" + os.devnull, "-o", "ConnectTimeout={}".format(self._config["ssh_timeout"]), - "-p", str(self.ssh_port), "-i", self._ssh_tmp_key_file] + "-p", str(self.ssh_port), "-i", self._ssh_tmp_key_file, + "-o", "IdentitiesOnly=yes"] # If not in debug mode, set ssh to quiet mode to # avoid printing the results of commands. if not self.debug: