diff --git a/.gitlab-ci.d/cirrus.yml b/.gitlab-ci.d/cirrus.yml index 07dc6edae1..64f2e25afa 100644 --- a/.gitlab-ci.d/cirrus.yml +++ b/.gitlab-ci.d/cirrus.yml @@ -74,6 +74,22 @@ aarch64-macos-13-base-build: PKG_CONFIG_PATH: /opt/homebrew/curl/lib/pkgconfig:/opt/homebrew/ncurses/lib/pkgconfig:/opt/homebrew/readline/lib/pkgconfig TEST_TARGETS: check-unit check-block check-qapi-schema check-softfloat check-qtest-x86_64 +aarch64-macos-14-base-build: + extends: .cirrus_build_job + variables: + NAME: macos-14 + CIRRUS_VM_INSTANCE_TYPE: macos_instance + CIRRUS_VM_IMAGE_SELECTOR: image + CIRRUS_VM_IMAGE_NAME: ghcr.io/cirruslabs/macos-sonoma-base:latest + CIRRUS_VM_CPUS: 12 + CIRRUS_VM_RAM: 24G + UPDATE_COMMAND: brew update + INSTALL_COMMAND: brew install + PATH_EXTRA: /opt/homebrew/ccache/libexec:/opt/homebrew/gettext/bin + PKG_CONFIG_PATH: /opt/homebrew/curl/lib/pkgconfig:/opt/homebrew/ncurses/lib/pkgconfig:/opt/homebrew/readline/lib/pkgconfig + TEST_TARGETS: check-unit check-block check-qapi-schema check-softfloat check-qtest-x86_64 + QEMU_JOB_OPTIONAL: 1 + # The following jobs run VM-based tests via KVM on a Linux-based Cirrus-CI job .cirrus_kvm_job: diff --git a/.gitlab-ci.d/cirrus/macos-14.vars b/.gitlab-ci.d/cirrus/macos-14.vars new file mode 100644 index 0000000000..43070f4a26 --- /dev/null +++ b/.gitlab-ci.d/cirrus/macos-14.vars @@ -0,0 +1,16 @@ +# THIS FILE WAS AUTO-GENERATED +# +# $ lcitool variables macos-14 qemu +# +# https://gitlab.com/libvirt/libvirt-ci + +CCACHE='/opt/homebrew/bin/ccache' +CPAN_PKGS='' +CROSS_PKGS='' +MAKE='/opt/homebrew/bin/gmake' +NINJA='/opt/homebrew/bin/ninja' +PACKAGING_COMMAND='brew' +PIP3='/opt/homebrew/bin/pip3' +PKGS='bash bc bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 jemalloc jpeg-turbo json-c libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson mtools ncurses nettle ninja pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy socat sparse spice-protocol swtpm tesseract usbredir vde vte3 xorriso zlib zstd' +PYPI_PKGS='PyYAML numpy pillow sphinx sphinx-rtd-theme tomli' +PYTHON='/opt/homebrew/bin/python3' diff --git a/audio/audio.c b/audio/audio.c index f91e05b72c..8d1e4ad922 100644 --- a/audio/audio.c +++ b/audio/audio.c @@ -1758,12 +1758,15 @@ static AudioState *audio_init(Audiodev *dev, Error **errp) goto out; } s->dev = dev = e->dev; + QSIMPLEQ_REMOVE_HEAD(&default_audiodevs, next); + g_free(e); drvname = AudiodevDriver_str(dev->driver); driver = audio_driver_lookup(drvname); if (!audio_driver_init(s, driver, dev, NULL)) { break; } - QSIMPLEQ_REMOVE_HEAD(&default_audiodevs, next); + qapi_free_Audiodev(dev); + s->dev = NULL; } } diff --git a/configure b/configure index 42fe4d0510..bdda912f36 100755 --- a/configure +++ b/configure @@ -41,12 +41,7 @@ then # This file is auto-generated by configure to support in-source tree # 'make' command invocation -ifeq ($(MAKECMDGOALS),) -recurse: all -endif - -.NOTPARALLEL: % -%: force +build: @echo 'changing dir to build for $(MAKE) "$(MAKECMDGOALS)"...' @$(MAKE) -C build -f Makefile $(MAKECMDGOALS) @if test "$(MAKECMDGOALS)" = "distclean" && \ @@ -54,8 +49,9 @@ endif then \ rm -rf build GNUmakefile ; \ fi -force: ; -.PHONY: force +%: build + @ +.PHONY: build GNUmakefile: ; EOF @@ -968,14 +964,14 @@ meson="$(cd pyvenv/bin; pwd)/meson" # Conditionally ensure Sphinx is installed. -mkvenv_flags="" -if test "$download" = "enabled" -a "$docs" = "enabled" ; then - mkvenv_flags="--online" +mkvenv_online_flag="" +if test "$download" = "enabled" ; then + mkvenv_online_flag=" --online" fi if test "$docs" != "disabled" ; then if ! $mkvenv ensuregroup \ - $mkvenv_flags \ + $(test "$docs" = "enabled" && echo "$mkvenv_online_flag") \ ${source_path}/pythondeps.toml docs; then if test "$docs" = "enabled" ; then @@ -1634,6 +1630,7 @@ if test "$container" != no; then fi echo "SUBDIRS=$subdirs" >> $config_host_mak echo "PYTHON=$python" >> $config_host_mak +echo "MKVENV_ENSUREGROUP=$mkvenv ensuregroup $mkvenv_online_flag" >> $config_host_mak echo "GENISOIMAGE=$genisoimage" >> $config_host_mak echo "MESON=$meson" >> $config_host_mak echo "NINJA=$ninja" >> $config_host_mak diff --git a/disas/cris.c b/disas/cris.c index 0b0a3fb916..409a224c5d 100644 --- a/disas/cris.c +++ b/disas/cris.c @@ -1731,10 +1731,10 @@ format_hex (unsigned long number, unsigned (== 0). */ static char * -format_dec (long number, char *outbuffer, int signedp) +format_dec (long number, char *outbuffer, size_t outsize, int signedp) { last_immediate = number; - sprintf (outbuffer, signedp ? "%ld" : "%lu", number); + snprintf (outbuffer, outsize, signedp ? "%ld" : "%lu", number); return outbuffer + strlen (outbuffer); } @@ -1876,6 +1876,12 @@ print_flags (struct cris_disasm_data *disdata, unsigned int insn, char *cp) return cp; } +#define FORMAT_DEC(number, tp, signedp) \ + format_dec (number, tp, ({ \ + assert(tp >= temp && tp <= temp + sizeof(temp)); \ + temp + sizeof(temp) - tp; \ + }), signedp) + /* Print out an insn with its operands, and update the info->insn_type fields. The prefix_opcodep and the rest hold a prefix insn that is supposed to be output as an address mode. */ @@ -2105,7 +2111,7 @@ print_with_operands (const struct cris_opcode *opcodep, if ((*cs == 'z' && (insn & 0x20)) || (opcodep->match == BDAP_QUICK_OPCODE && (nbytes <= 2 || buffer[1 + nbytes] == 0))) - tp = format_dec (number, tp, signedp); + tp = FORMAT_DEC (number, tp, signedp); else { unsigned int highbyte = (number >> 24) & 0xff; @@ -2241,7 +2247,7 @@ print_with_operands (const struct cris_opcode *opcodep, with_reg_prefix); if (number >= 0) *tp++ = '+'; - tp = format_dec (number, tp, 1); + tp = FORMAT_DEC (number, tp, 1); info->flags |= CRIS_DIS_FLAG_MEM_TARGET_IS_REG; info->target = (prefix_insn >> 12) & 15; @@ -2340,7 +2346,7 @@ print_with_operands (const struct cris_opcode *opcodep, { if (number >= 0) *tp++ = '+'; - tp = format_dec (number, tp, 1); + tp = FORMAT_DEC (number, tp, 1); } } else @@ -2397,7 +2403,7 @@ print_with_operands (const struct cris_opcode *opcodep, break; case 'I': - tp = format_dec (insn & 63, tp, 0); + tp = FORMAT_DEC (insn & 63, tp, 0); break; case 'b': @@ -2426,11 +2432,11 @@ print_with_operands (const struct cris_opcode *opcodep, break; case 'c': - tp = format_dec (insn & 31, tp, 0); + tp = FORMAT_DEC (insn & 31, tp, 0); break; case 'C': - tp = format_dec (insn & 15, tp, 0); + tp = FORMAT_DEC (insn & 15, tp, 0); break; case 'o': @@ -2463,7 +2469,7 @@ print_with_operands (const struct cris_opcode *opcodep, if (number > 127) number = number - 256; - tp = format_dec (number, tp, 1); + tp = FORMAT_DEC (number, tp, 1); *tp++ = ','; tp = format_reg (disdata, (insn >> 12) & 15, tp, with_reg_prefix); } @@ -2474,7 +2480,7 @@ print_with_operands (const struct cris_opcode *opcodep, break; case 'i': - tp = format_dec ((insn & 32) ? (insn & 31) | ~31L : insn & 31, tp, 1); + tp = FORMAT_DEC ((insn & 32) ? (insn & 31) | ~31L : insn & 31, tp, 1); break; case 'P': diff --git a/docs/devel/build-system.rst b/docs/devel/build-system.rst index 21f78da7d1..43d6005881 100644 --- a/docs/devel/build-system.rst +++ b/docs/devel/build-system.rst @@ -122,10 +122,78 @@ functioning. These are performed using a few more helper functions: indicated by $TMPC. -Python virtual environments and the QEMU build system ------------------------------------------------------ +Python virtual environments and the build process +------------------------------------------------- + +An important step in ``configure`` is to create a Python virtual +environment (venv) during the configuration phase. The Python interpreter +comes from the ``--python`` command line option, the ``$PYTHON`` variable +from the environment, or the system PATH, in this order. The venv resides +in the ``pyvenv`` directory in the build tree, and provides consistency +in how the build process runs Python code. + +At this stage, ``configure`` also queries the chosen Python interpreter +about QEMU's build dependencies. Note that the build process does *not* +look for ``meson``, ``sphinx-build`` or ``avocado`` binaries in the PATH; +likewise, there are no options such as ``--meson`` or ``--sphinx-build``. +This avoids a potential mismatch, where Meson and Sphinx binaries on the +PATH might operate in a different Python environment than the one chosen +by the user during the build process. On the other hand, it introduces +a potential source of confusion where the user installs a dependency but +``configure`` is not able to find it. When this happens, the dependency +was installed in the ``site-packages`` directory of another interpreter, +or with the wrong ``pip`` program. + +If a package is available for the chosen interpreter, ``configure`` +prepares a small script that invokes it from the venv itself[#distlib]_. +If not, ``configure`` can also optionally install dependencies in the +virtual environment with ``pip``, either from wheels in ``python/wheels`` +or by downloading the package with PyPI. Downloading can be disabled with +``--disable-download``; and anyway, it only happens when a ``configure`` +option (currently, only ``--enable-docs``) is explicitly enabled but +the dependencies are not present[#pip]_. + +.. [#distlib] The scripts are created based on the package's metadata, + specifically the ``console_script`` entry points. This is the + same mechanism that ``pip`` uses when installing a package. + Currently, in all cases it would be possible to use ``python -m`` + instead of an entry point script, which makes this approach a + bit overkill. On the other hand, creating the scripts is + future proof and it makes the contents of the ``pyvenv/bin`` + directory more informative. Portability is also not an issue, + because the Python Packaging Authority provides a package + ``distlib.scripts`` to perform this task. + +.. [#pip] ``pip`` might also be used when running ``make check-avocado`` + if downloading is enabled, to ensure that Avocado is + available. + +The required versions of the packages are stored in a configuration file +``pythondeps.toml``. The format is custom to QEMU, but it is documented +at the top of the file itself and it should be easy to understand. The +requirements should make it possible to use the version that is packaged +that is provided by supported distros. + +When dependencies are downloaded, instead, ``configure`` uses a "known +good" version that is also listed in ``pythondeps.toml``. In this +scenario, ``pythondeps.toml`` behaves like the "lock file" used by +``cargo``, ``poetry`` or other dependency management systems. + + +Bundled Python packages +----------------------- + +Python packages that are **mandatory** dependencies to build QEMU, +but are not available in all supported distros, are bundled with the +QEMU sources. Currently this includes Meson (outdated in CentOS 8 +and derivatives, Ubuntu 20.04 and 22.04, and openSUSE Leap) and tomli +(absent in Ubuntu 20.04). + +If you need to update these, please do so by modifying and rerunning +``python/scripts/vendor.py``. This script embeds the sha256 hash of +package sources and checks it. The pypi.org web site provides an easy +way to retrieve the sha256 hash of the sources. -TBD Stage 2: Meson ============== @@ -376,6 +444,15 @@ This is needed to obey the --python= option passed to the configure script, which may point to something other than the first python3 binary on the path. +By the time Meson runs, Python dependencies are available in the virtual +environment and should be invoked through the scripts that ``configure`` +places under ``pyvenv``. One way to do so is as follows, using Meson's +``find_program`` function:: + + sphinx_build = find_program( + fs.parent(python.full_path()) / 'sphinx-build', + required: get_option('docs')) + Stage 3: Make ============= @@ -434,6 +511,11 @@ number of dynamically created files listed later. executables. Build rules for various subdirectories are included in other meson.build files spread throughout the QEMU source tree. +``python/scripts/mkvenv.py`` + A wrapper for the Python ``venv`` and ``distlib.scripts`` packages. + It handles creating the virtual environment, creating scripts in + ``pyvenv/bin``, and calling ``pip`` to install dependencies. + ``tests/Makefile.include`` Rules for external test harnesses. These include the TCG tests and the Avocado-based integration tests. diff --git a/python/scripts/vendor.py b/python/scripts/vendor.py index 7627487117..1038b14ae0 100755 --- a/python/scripts/vendor.py +++ b/python/scripts/vendor.py @@ -41,8 +41,8 @@ def main() -> int: parser.parse_args() packages = { - "meson==0.63.3": - "d677b809c4895dcbaac9bf6c43703fcb3609a4b24c6057c78f828590049cf43a", + "meson==1.2.3": + "4533a43c34548edd1f63a276a42690fce15bde9409bcf20c4b8fa3d7e4d7cac1", "tomli==2.0.1": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", diff --git a/python/wheels/meson-0.63.3-py3-none-any.whl b/python/wheels/meson-0.63.3-py3-none-any.whl deleted file mode 100644 index 8a191e3a20..0000000000 Binary files a/python/wheels/meson-0.63.3-py3-none-any.whl and /dev/null differ diff --git a/python/wheels/meson-1.2.3-py3-none-any.whl b/python/wheels/meson-1.2.3-py3-none-any.whl new file mode 100644 index 0000000000..a8b84e5f11 Binary files /dev/null and b/python/wheels/meson-1.2.3-py3-none-any.whl differ diff --git a/pythondeps.toml b/pythondeps.toml index 0a35ebcf9f..0e88415999 100644 --- a/pythondeps.toml +++ b/pythondeps.toml @@ -10,7 +10,8 @@ # - accepted: accepted versions when using a system package # - installed: fixed version to install in the virtual environment # if a system package is not found; if not specified, -# the minimum and maximum +# defaults to the same as "accepted" or, if also missing, +# to the newest version available on PyPI. # - canary: if specified, use this program name to present more # precise error diagnostics to the user. For example, # 'sphinx-build' can be used as a bellwether for the @@ -18,7 +19,7 @@ [meson] # The install key should match the version in python/wheels/ -meson = { accepted = ">=0.63.0", installed = "0.63.3", canary = "meson" } +meson = { accepted = ">=0.63.0", installed = "1.2.3", canary = "meson" } [docs] sphinx = { accepted = ">=1.6", installed = "5.3.0", canary = "sphinx-build" } diff --git a/scripts/coverity-scan/model.c b/scripts/coverity-scan/model.c index 686d1a3008..a064d84084 100644 --- a/scripts/coverity-scan/model.c +++ b/scripts/coverity-scan/model.c @@ -42,94 +42,6 @@ typedef _Bool bool; typedef struct va_list_str *va_list; -/* exec.c */ - -typedef struct AddressSpace AddressSpace; -typedef struct MemoryRegionCache MemoryRegionCache; -typedef uint64_t hwaddr; -typedef uint32_t MemTxResult; -typedef struct MemTxAttrs {} MemTxAttrs; - -static void __bufwrite(uint8_t *buf, ssize_t len) -{ - int first, last; - __coverity_negative_sink__(len); - if (len == 0) return; - buf[0] = first; - buf[len-1] = last; - __coverity_writeall__(buf); -} - -static void __bufread(uint8_t *buf, ssize_t len) -{ - __coverity_negative_sink__(len); - if (len == 0) return; - int first = buf[0]; - int last = buf[len-1]; -} - -MemTxResult address_space_read_cached(MemoryRegionCache *cache, hwaddr addr, - MemTxAttrs attrs, - void *buf, int len) -{ - MemTxResult result; - // TODO: investigate impact of treating reads as producing - // tainted data, with __coverity_tainted_data_argument__(buf). - __bufwrite(buf, len); - return result; -} - -MemTxResult address_space_write_cached(MemoryRegionCache *cache, hwaddr addr, - MemTxAttrs attrs, - const void *buf, int len) -{ - MemTxResult result; - __bufread(buf, len); - return result; -} - -MemTxResult address_space_rw_cached(MemoryRegionCache *cache, hwaddr addr, - MemTxAttrs attrs, - void *buf, int len, bool is_write) -{ - if (is_write) { - return address_space_write_cached(cache, addr, attrs, buf, len); - } else { - return address_space_read_cached(cache, addr, attrs, buf, len); - } -} - -MemTxResult address_space_read(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, - void *buf, int len) -{ - MemTxResult result; - // TODO: investigate impact of treating reads as producing - // tainted data, with __coverity_tainted_data_argument__(buf). - __bufwrite(buf, len); - return result; -} - -MemTxResult address_space_write(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, - const void *buf, int len) -{ - MemTxResult result; - __bufread(buf, len); - return result; -} - -MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, - void *buf, int len, bool is_write) -{ - if (is_write) { - return address_space_write(as, addr, attrs, buf, len); - } else { - return address_space_read(as, addr, attrs, buf, len); - } -} - /* Tainting */ typedef struct {} name2keysym_t; diff --git a/scripts/coverity-scan/run-coverity-scan b/scripts/coverity-scan/run-coverity-scan index 129672c86f..d56c9b6677 100755 --- a/scripts/coverity-scan/run-coverity-scan +++ b/scripts/coverity-scan/run-coverity-scan @@ -116,14 +116,14 @@ update_coverity_tools () { cd "$COVERITY_TOOL_BASE" echo "Checking for new version of coverity build tools..." - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME&md5=1" -O coverity_tool.md5.new + wget https://scan.coverity.com/download/cxx/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME&md5=1" -O coverity_tool.md5.new if ! cmp -s coverity_tool.md5 coverity_tool.md5.new; then # out of date md5 or no md5: download new build tool # blow away the old build tool echo "Downloading coverity build tools..." rm -rf coverity_tool coverity_tool.tgz - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -O coverity_tool.tgz + wget https://scan.coverity.com/download/cxx/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -O coverity_tool.tgz if ! (cat coverity_tool.md5.new; echo " coverity_tool.tgz") | md5sum -c --status; then echo "Downloaded tarball didn't match md5sum!" exit 1 diff --git a/system/memory.c b/system/memory.c index 304fa843ea..4d9cb0a7ff 100644 --- a/system/memory.c +++ b/system/memory.c @@ -1692,7 +1692,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr, /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL. */ assert(ptr != NULL); - mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal); + mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort); } void memory_region_init_ram_device_ptr(MemoryRegion *mr, @@ -1711,7 +1711,7 @@ void memory_region_init_ram_device_ptr(MemoryRegion *mr, /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL. */ assert(ptr != NULL); - mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal); + mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort); } void memory_region_init_alias(MemoryRegion *mr, diff --git a/system/physmem.c b/system/physmem.c index fc2b0fee01..a63853a7bc 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -2699,6 +2699,17 @@ static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr, l = memory_access_size(mr, l, addr1); /* XXX: could force current_cpu to NULL to avoid potential bugs */ + + /* + * Assure Coverity (and ourselves) that we are not going to OVERRUN + * the buffer by following ldn_he_p(). + */ +#ifdef QEMU_STATIC_ANALYSIS + assert((l == 1 && len >= 1) || + (l == 2 && len >= 2) || + (l == 4 && len >= 4) || + (l == 8 && len >= 8)); +#endif val = ldn_he_p(buf, l); result |= memory_region_dispatch_write(mr, addr1, val, size_memop(l), attrs); @@ -2769,6 +2780,17 @@ MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr, l = memory_access_size(mr, l, addr1); result |= memory_region_dispatch_read(mr, addr1, &val, size_memop(l), attrs); + + /* + * Assure Coverity (and ourselves) that we are not going to OVERRUN + * the buffer by following stn_he_p(). + */ +#ifdef QEMU_STATIC_ANALYSIS + assert((l == 1 && len >= 1) || + (l == 2 && len >= 2) || + (l == 4 && len >= 4) || + (l == 8 && len >= 8)); +#endif stn_he_p(buf, l, val); } else { /* RAM case */ diff --git a/tests/Makefile.include b/tests/Makefile.include index dab1989a07..c9d1674bd0 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -111,7 +111,7 @@ quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \ $(TESTS_VENV_TOKEN): $(SRC_PATH)/pythondeps.toml $(call quiet-venv-pip,install -e "$(SRC_PATH)/python/") - $(PYTHON) python/scripts/mkvenv.py ensuregroup --online $< avocado + $(MKVENV_ENSUREGROUP) $< avocado $(call quiet-command, touch $@) $(TESTS_RESULTS_DIR): diff --git a/tests/lcitool/libvirt-ci b/tests/lcitool/libvirt-ci index 36bc517161..77c800186f 160000 --- a/tests/lcitool/libvirt-ci +++ b/tests/lcitool/libvirt-ci @@ -1 +1 @@ -Subproject commit 36bc517161c45ead20224d47f2dc4fa428af6724 +Subproject commit 77c800186f34b21be7660750577cc5582a914deb diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh index 993683cf48..0c93557ad6 100755 --- a/tests/lcitool/refresh +++ b/tests/lcitool/refresh @@ -207,6 +207,7 @@ try: # generate_cirrus("freebsd-13") generate_cirrus("macos-13") + generate_cirrus("macos-14") # # VM packages lists