OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity

docs/devel: rationalise unstable gitlab tests under FLAKY_TESTS

Browse Source 
It doesn't make sense to have two classes of flaky tests. While it may
take the constrained environment of CI to trigger failures easily it
doesn't mean they don't occasionally happen on developer machines. As
CI is the gating factor to passing there is no point developers
running the tests locally anyway unless they are trying to fix things.

While we are at it update the language in the docs to discourage the
QEMU_TEST_FLAKY_TESTS becoming a permanent solution.

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
Acked-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
Message-Id: <20231201093633.2551497-3-alex.bennee@linaro.org>
This commit is contained in:
Alex Bennée 2023-12-01 09:36:28 +00:00
parent 5dcf6334ae
commit 9b45cc9931
13 changed files with 71 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
docs/devel/testing.rst
View File

@ -1371,23 +1371,32 @@ conditions. For example, tests that take longer to execute when QEMU is
compiled with debug flags. Therefore, the ``AVOCADO_TIMEOUT_EXPECTED`` variable
has been used to determine whether those tests should run or not.
GITLAB_CI
^^^^^^^^^
A number of tests are flagged to not run on the GitLab CI. Usually because
they proved to the flaky or there are constraints on the CI environment which
would make them fail. If you encounter a similar situation then use that
variable as shown on the code snippet below to skip the test:
.. code::
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
def test(self):
do_something()
QEMU_TEST_FLAKY_TESTS
^^^^^^^^^^^^^^^^^^^^^
Some tests are not working reliably and thus are disabled by default.
Set this environment variable to enable them.
This includes tests that don't run reliably on GitLab's CI which
usually expose real issues that are rarely seen on developer machines
due to the constraints of the CI environment. If you encounter a
similar situation then raise a bug and then mark the test as shown on
the code snippet below:
.. code::
# See https://gitlab.com/qemu-project/qemu/-/issues/nnnn
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test(self):
do_something()
Tests should not live in this state forever and should either be fixed
or eventually removed.
To run such tests locally you will need to set the environment
variable. For example:
.. code::
env QEMU_TEST_FLAKY_TESTS=1 ./pyvenv/bin/avocado run \
tests/avocado/boot_linux.py:BootLinuxPPC64.test_pseries_tcg
Uninstalling Avocado
~~~~~~~~~~~~~~~~~~~~

8
tests/avocado/boot_linux.py
View File

@ -12,7 +12,7 @@ import os
from avocado_qemu import LinuxTest, BUILD_DIR
from avocado import skipIf
from avocado import skipUnless
class BootLinuxX8664(LinuxTest):
@ -93,7 +93,8 @@ class BootLinuxPPC64(LinuxTest):
timeout = 360
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_pseries_tcg(self):
"""
:avocado: tags=machine:pseries
@ -111,7 +112,8 @@ class BootLinuxS390X(LinuxTest):
timeout = 240
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_s390_ccw_virtio_tcg(self):
"""
:avocado: tags=machine:s390-ccw-virtio

5
tests/avocado/boot_linux_console.py
View File

@ -15,7 +15,7 @@ import shutil
from avocado import skip
from avocado import skipUnless
from avocado import skipIf
from avocado import skipUnless
from avocado_qemu import QemuSystemTest
from avocado_qemu import exec_command
from avocado_qemu import exec_command_and_wait_for_pattern
@ -1419,7 +1419,8 @@ class BootLinuxConsole(LinuxKernelTest):
# This test has a 6-10% failure rate on various hosts that look
# like issues with a buggy kernel. As a result we don't want it
# gating releases on Gitlab.
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_sh4_r2d(self):
"""
:avocado: tags=arch:sh4

5
tests/avocado/intel_iommu.py
View File

@ -9,10 +9,11 @@
# later. See the COPYING file in the top-level directory.
import os
from avocado import skipIf
from avocado import skipUnless
from avocado_qemu import LinuxTest
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
class IntelIOMMU(LinuxTest):
"""
:avocado: tags=arch:x86_64

5
tests/avocado/linux_initrd.py
View File

@ -13,7 +13,7 @@ import logging
import tempfile
from avocado_qemu import QemuSystemTest
from avocado import skipIf
from avocado import skipUnless
class LinuxInitrd(QemuSystemTest):
@ -53,7 +53,8 @@ class LinuxInitrd(QemuSystemTest):
max_size + 1)
self.assertRegex(self.vm.get_log(), expected_msg)
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_with_2gib_file_should_work_with_linux_v4_16(self):
"""
QEMU has supported up to 4 GiB initrd for recent kernel

8
tests/avocado/machine_aspeed.py
View File

@ -18,7 +18,7 @@ from avocado_qemu import exec_command_and_wait_for_pattern
from avocado_qemu import interrupt_interactive_console_until_pattern
from avocado_qemu import has_cmd
from avocado.utils import archive
from avocado import skipIf
from avocado import skipUnless
from avocado import skipUnless
@ -311,7 +311,8 @@ class AST2x00MachineSDK(QemuSystemTest, LinuxSSHMixIn):
self, 'boot', '## Loading kernel from FIT Image')
self.wait_for_console_pattern('Starting kernel ...')
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_arm_ast2500_evb_sdk(self):
"""
:avocado: tags=arch:arm
@ -329,7 +330,8 @@ class AST2x00MachineSDK(QemuSystemTest, LinuxSSHMixIn):
self.workdir + '/ast2500-default/image-bmc')
self.wait_for_console_pattern('nodistro.0 ast2500-default ttyS4')
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_arm_ast2600_evb_sdk(self):
"""
:avocado: tags=arch:arm

8
tests/avocado/machine_mips_malta.py
View File

@ -11,7 +11,7 @@ import os
import gzip
import logging
from avocado import skipIf
from avocado import skipUnless
from avocado import skipUnless
from avocado.utils import archive
from avocado_qemu import QemuSystemTest
@ -101,7 +101,8 @@ class MaltaMachineFramebuffer(QemuSystemTest):
"""
self.do_test_i6400_framebuffer_logo(1)
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_mips_malta_i6400_framebuffer_logo_7cores(self):
"""
:avocado: tags=arch:mips64el
@ -111,7 +112,8 @@ class MaltaMachineFramebuffer(QemuSystemTest):
"""
self.do_test_i6400_framebuffer_logo(7)
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_mips_malta_i6400_framebuffer_logo_8cores(self):
"""
:avocado: tags=arch:mips64el

8
tests/avocado/machine_rx_gdbsim.py
View File

@ -10,7 +10,7 @@
import os
from avocado import skipIf
from avocado import skipUnless
from avocado_qemu import QemuSystemTest
from avocado_qemu import exec_command_and_wait_for_pattern
from avocado_qemu import wait_for_console_pattern
@ -22,7 +22,8 @@ class RxGdbSimMachine(QemuSystemTest):
timeout = 30
KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 '
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_uboot(self):
"""
U-Boot and checks that the console is operational.
@ -46,7 +47,8 @@ class RxGdbSimMachine(QemuSystemTest):
# FIXME limit baudrate on chardev, else we type too fast
#exec_command_and_wait_for_pattern(self, 'version', gcc_version)
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_linux_sash(self):
"""
Boots a Linux kernel and checks that the console is operational.

2
tests/avocado/machine_s390_ccw_virtio.py
View File

@ -12,7 +12,7 @@
import os
import tempfile
from avocado import skipIf
from avocado import skipUnless
from avocado_qemu import QemuSystemTest
from avocado_qemu import exec_command_and_wait_for_pattern
from avocado_qemu import wait_for_console_pattern

6
tests/avocado/replay_kernel.py
View File

@ -15,7 +15,7 @@ import logging
import time
from avocado import skip
from avocado import skipIf
from avocado import skipUnless
from avocado import skipUnless
from avocado_qemu import wait_for_console_pattern
from avocado.utils import archive
@ -82,6 +82,7 @@ class ReplayKernelBase(LinuxKernelTest):
class ReplayKernelNormal(ReplayKernelBase):
# See https://gitlab.com/qemu-project/qemu/-/issues/2010
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test sometimes gets stuck')
def test_x86_64_pc(self):
"""
@ -179,7 +180,8 @@ class ReplayKernelNormal(ReplayKernelBase):
self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=1)
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_arm_cubieboard_initrd(self):
"""
:avocado: tags=arch:arm

14
tests/avocado/reverse_debugging.py
View File

@ -10,7 +10,7 @@
import os
import logging
from avocado import skipIf
from avocado import skipUnless
from avocado_qemu import BUILD_DIR
from avocado.utils import datadrainer
from avocado.utils import gdb
@ -206,7 +206,8 @@ class ReverseDebugging_X86_64(ReverseDebugging):
+ self.get_reg_le(g, self.REG_CS) * 0x10
# unidentified gitlab timeout problem
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_x86_64_pc(self):
"""
:avocado: tags=arch:x86_64
@ -223,7 +224,8 @@ class ReverseDebugging_AArch64(ReverseDebugging):
REG_PC = 32
# unidentified gitlab timeout problem
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_aarch64_virt(self):
"""
:avocado: tags=arch:aarch64
@ -247,7 +249,8 @@ class ReverseDebugging_ppc64(ReverseDebugging):
REG_PC = 0x40
# unidentified gitlab timeout problem
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_ppc64_pseries(self):
"""
:avocado: tags=arch:ppc64
@ -260,7 +263,8 @@ class ReverseDebugging_ppc64(ReverseDebugging):
self.reverse_debugging()
# See https://gitlab.com/qemu-project/qemu/-/issues/1992
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_ppc64_powernv(self):
"""
:avocado: tags=arch:ppc64

5
tests/avocado/smmu.py
View File

@ -9,10 +9,11 @@
# later. See the COPYING file in the top-level directory.
import os
from avocado import skipIf
from avocado import skipUnless
from avocado_qemu import LinuxTest, BUILD_DIR
@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
class SMMU(LinuxTest):
"""
:avocado: tags=accel:kvm

4
tests/avocado/tuxrun_baselines.py
View File

@ -13,7 +13,7 @@ import os
import time
import tempfile
from avocado import skip, skipIf
from avocado import skip, skipUnless
from avocado_qemu import QemuSystemTest
from avocado_qemu import exec_command, exec_command_and_wait_for_pattern
from avocado_qemu import wait_for_console_pattern
@ -551,7 +551,7 @@ class TuxRunBaselineTest(QemuSystemTest):
haltmsg="Requesting system halt")
# Note: some segfaults caused by unaligned userspace access
@skipIf(os.getenv('GITLAB_CI'), 'Skipping unstable test on GitLab')
@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
def test_sh4(self):
"""
:avocado: tags=arch:sh4