eaf245becd
This reverts commit ec5ffa0056.
Bumping avocado to version 101 has two issues. First, there are problems
where Avocado is not logging of command lines or terminal output, and not
collecting Python logs outside the avocado namespace.
Second, the recent changes to Python handling mean that there is a single
virtual environment for all the build, instead of a separate one for testing.
Requiring a too-new version of avocado causes conflicts with any avocado
plugins installed on the host:
$ make check-venv
make[1]: Entering directory '/home/berrange/src/virt/qemu/build'
GIT ui/keycodemapdb tests/fp/berkeley-testfloat-3 tests/fp/berkeley-softfloat-3 dtc
VENVPIP install -e /home/berrange/src/virt/qemu/python/
VENVPIP install -r /home/berrange/src/virt/qemu/tests/requirements.txt
ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
avocado-framework-plugin-varianter-yaml-to-mux 98.0 requires avocado-framework==98.0, but you have avocado-framework 101.0 which is incompatible.
avocado-framework-plugin-result-html 98.0 requires avocado-framework==98.0, but you have avocado-framework 101.0 which is incompatible.
make[1]: Leaving directory '/home/berrange/src/virt/qemu/build'
To avoid this issue, tests/requirements.txt should use a ">=" constraint
and the version of Avocado should be limited to what distros provide
in the system packages. Only Fedora has Avocado, and more specifically
version 92.0 (though 98.0 is also available as a module). As a first
step, this patch reverts the introduction of a too-new Avocado.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
170 lines
6.2 KiB
Makefile
170 lines
6.2 KiB
Makefile
# -*- Mode: makefile -*-
|
|
|
|
.PHONY: check-help
|
|
check-help:
|
|
@echo "Regression testing targets:"
|
|
@echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests"
|
|
@echo " $(MAKE) bench Run speed tests"
|
|
@echo
|
|
@echo "Individual test suites:"
|
|
@echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target"
|
|
@echo " $(MAKE) check-qtest Run qtest tests"
|
|
@echo " $(MAKE) check-unit Run qobject tests"
|
|
@echo " $(MAKE) check-qapi-schema Run QAPI schema tests"
|
|
@echo " $(MAKE) check-block Run block tests"
|
|
ifneq ($(filter $(all-check-targets), check-softfloat),)
|
|
@echo " $(MAKE) check-tcg Run TCG tests"
|
|
@echo " $(MAKE) check-softfloat Run FPU emulation tests"
|
|
endif
|
|
@echo " $(MAKE) check-avocado Run avocado (integration) tests for currently configured targets"
|
|
@echo
|
|
@echo " $(MAKE) check-report.junit.xml Generates an aggregated XML test report"
|
|
@echo " $(MAKE) check-venv Creates a Python venv for tests"
|
|
@echo " $(MAKE) check-clean Clean the tests and related data"
|
|
@echo
|
|
@echo "The following are useful for CI builds"
|
|
@echo " $(MAKE) check-build Build most test binaries"
|
|
@echo " $(MAKE) get-vm-images Downloads all images used by avocado tests, according to configured targets (~350 MB each, 1.5 GB max)"
|
|
@echo
|
|
@echo
|
|
@echo "The variable SPEED can be set to control the gtester speed setting."
|
|
@echo "Default options are -k and (for $(MAKE) V=1) --verbose; they can be"
|
|
@echo "changed with variable GTESTER_OPTIONS."
|
|
|
|
ifneq ($(wildcard config-host.mak),)
|
|
export SRC_PATH
|
|
|
|
SPEED = quick
|
|
|
|
# Per guest TCG tests
|
|
BUILD_TCG_TARGET_RULES=$(patsubst %,build-tcg-tests-%, $(TCG_TESTS_TARGETS))
|
|
CLEAN_TCG_TARGET_RULES=$(patsubst %,clean-tcg-tests-%, $(TCG_TESTS_TARGETS))
|
|
DISTCLEAN_TCG_TARGET_RULES=$(patsubst %,distclean-tcg-tests-%, $(TCG_TESTS_TARGETS))
|
|
RUN_TCG_TARGET_RULES=$(patsubst %,run-tcg-tests-%, $(TCG_TESTS_TARGETS))
|
|
|
|
$(foreach TARGET,$(TCG_TESTS_TARGETS), \
|
|
$(eval $(BUILD_DIR)/tests/tcg/config-$(TARGET).mak: config-host.mak))
|
|
|
|
.PHONY: $(TCG_TESTS_TARGETS:%=build-tcg-tests-%)
|
|
$(TCG_TESTS_TARGETS:%=build-tcg-tests-%): build-tcg-tests-%: $(BUILD_DIR)/tests/tcg/config-%.mak
|
|
$(call quiet-command, \
|
|
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS), \
|
|
"BUILD","$* guest-tests")
|
|
|
|
.PHONY: $(TCG_TESTS_TARGETS:%=run-tcg-tests-%)
|
|
$(TCG_TESTS_TARGETS:%=run-tcg-tests-%): run-tcg-tests-%: build-tcg-tests-%
|
|
$(call quiet-command, \
|
|
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) SPEED=$(SPEED) run, \
|
|
"RUN", "$* guest-tests")
|
|
|
|
.PHONY: $(TCG_TESTS_TARGETS:%=clean-tcg-tests-%)
|
|
$(TCG_TESTS_TARGETS:%=clean-tcg-tests-%): clean-tcg-tests-%:
|
|
$(call quiet-command, \
|
|
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) clean, \
|
|
"CLEAN", "$* guest-tests")
|
|
|
|
.PHONY: $(TCG_TESTS_TARGETS:%=distclean-tcg-tests-%)
|
|
$(TCG_TESTS_TARGETS:%=distclean-tcg-tests-%): distclean-tcg-tests-%:
|
|
$(call quiet-command, \
|
|
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) distclean, \
|
|
"CLEAN", "$* guest-tests")
|
|
|
|
.PHONY: build-tcg
|
|
build-tcg: $(BUILD_TCG_TARGET_RULES)
|
|
|
|
.PHONY: check-tcg
|
|
.ninja-goals.check-tcg = all $(if $(CONFIG_PLUGIN),test-plugins)
|
|
check-tcg: $(RUN_TCG_TARGET_RULES)
|
|
|
|
.PHONY: clean-tcg
|
|
clean-tcg: $(CLEAN_TCG_TARGET_RULES)
|
|
|
|
.PHONY: distclean-tcg
|
|
distclean-tcg: $(DISTCLEAN_TCG_TARGET_RULES)
|
|
|
|
# Python venv for running tests
|
|
|
|
.PHONY: check-venv check-avocado check-acceptance check-acceptance-deprecated-warning
|
|
|
|
# Build up our target list from the filtered list of ninja targets
|
|
TARGETS=$(patsubst libqemu-%.fa, %, $(filter libqemu-%.fa, $(ninja-targets)))
|
|
|
|
TESTS_VENV_TOKEN=$(BUILD_DIR)/pyvenv/tests.group
|
|
TESTS_VENV_REQ=$(SRC_PATH)/tests/requirements.txt
|
|
TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results
|
|
ifndef AVOCADO_TESTS
|
|
AVOCADO_TESTS=tests/avocado
|
|
endif
|
|
# Controls the output generated by Avocado when running tests.
|
|
# Any number of command separated loggers are accepted. For more
|
|
# information please refer to "avocado --help".
|
|
AVOCADO_SHOW=app
|
|
ifndef AVOCADO_TAGS
|
|
AVOCADO_CMDLINE_TAGS=$(patsubst %-softmmu,-t arch:%, \
|
|
$(filter %-softmmu,$(TARGETS)))
|
|
else
|
|
AVOCADO_CMDLINE_TAGS=$(addprefix -t , $(AVOCADO_TAGS))
|
|
endif
|
|
|
|
quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \
|
|
$(PYTHON) -m pip -q --disable-pip-version-check $1, \
|
|
"VENVPIP","$1")
|
|
|
|
$(TESTS_VENV_TOKEN): $(TESTS_VENV_REQ)
|
|
$(call quiet-venv-pip,install -e "$(SRC_PATH)/python/")
|
|
$(call quiet-venv-pip,install -r $(TESTS_VENV_REQ))
|
|
$(call quiet-command, touch $@)
|
|
|
|
$(TESTS_RESULTS_DIR):
|
|
$(call quiet-command, mkdir -p $@, \
|
|
MKDIR, $@)
|
|
|
|
check-venv: $(TESTS_VENV_TOKEN)
|
|
|
|
FEDORA_31_ARCHES_TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGETS)))
|
|
FEDORA_31_ARCHES_CANDIDATES=$(patsubst ppc64,ppc64le,$(FEDORA_31_ARCHES_TARGETS))
|
|
FEDORA_31_ARCHES := x86_64 aarch64 ppc64le s390x
|
|
FEDORA_31_DOWNLOAD=$(filter $(FEDORA_31_ARCHES),$(FEDORA_31_ARCHES_CANDIDATES))
|
|
|
|
# download one specific Fedora 31 image
|
|
get-vm-image-fedora-31-%: check-venv
|
|
$(call quiet-command, \
|
|
$(PYTHON) -m avocado vmimage get \
|
|
--distro=fedora --distro-version=31 --arch=$*, \
|
|
"AVOCADO", "Downloading avocado tests VM image for $*")
|
|
|
|
# download all vm images, according to defined targets
|
|
get-vm-images: check-venv $(patsubst %,get-vm-image-fedora-31-%, $(FEDORA_31_DOWNLOAD))
|
|
|
|
check-avocado: check-venv $(TESTS_RESULTS_DIR) get-vm-images
|
|
$(call quiet-command, \
|
|
$(PYTHON) -m avocado \
|
|
--show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \
|
|
$(if $(AVOCADO_TAGS),, --filter-by-tags-include-empty \
|
|
--filter-by-tags-include-empty-key) \
|
|
$(AVOCADO_CMDLINE_TAGS) \
|
|
$(if $(GITLAB_CI),,--failfast) $(AVOCADO_TESTS), \
|
|
"AVOCADO", "tests/avocado")
|
|
|
|
check-acceptance-deprecated-warning:
|
|
@echo
|
|
@echo "Note '$(MAKE) check-acceptance' is deprecated, use '$(MAKE) check-avocado' instead."
|
|
@echo
|
|
|
|
check-acceptance: check-acceptance-deprecated-warning | check-avocado
|
|
|
|
# Consolidated targets
|
|
|
|
.PHONY: check check-clean get-vm-images
|
|
check:
|
|
|
|
check-build: run-ninja
|
|
|
|
check-clean:
|
|
rm -rf $(TESTS_RESULTS_DIR)
|
|
|
|
clean: check-clean clean-tcg
|
|
distclean: distclean-tcg
|
|
|
|
endif
|