python: add mkvenv.py

This script will be responsible for building a lightweight Python
virtual environment at configure time. It works with Python 3.6 or
newer.

It has been designed to:
- work *offline*, no PyPI required.
- work *quickly*, The fast path is only ~65ms on my machine.
- work *robustly*, with multiple fallbacks to keep things working.
- work *cooperatively*, using system packages where possible.
  (You can use your distro's meson, no problem.)

Due to its unique position in the build chain, it exists outside of the
installable python packages in-tree and *must* be runnable without any
third party dependencies.

Under normal circumstances, the only dependency required to execute this
script is Python 3.6+ itself. The script is *faster* by several seconds
when setuptools and pip are installed in the host environment, which is
probably the case for a typical multi-purpose developer workstation.

In the event that pip/setuptools are missing or not usable, additional
dependencies may be required on some distributions which remove certain
Python stdlib modules to package them separately:

- Debian may require python3-venv to provide "ensurepip"
- NetBSD may require py310-expat to provide "pyexpat" *
  (* Or whichever version is current for NetBSD.)

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: John Snow <jsnow@redhat.com>
Message-Id: <20230511035435.734312-4-jsnow@redhat.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
This commit is contained in:
John Snow 2023-05-10 23:54:11 -04:00 committed by Paolo Bonzini
parent 6c2537d35b
commit dd84028ff9
6 changed files with 260 additions and 0 deletions

247
python/scripts/mkvenv.py Normal file
View File

@ -0,0 +1,247 @@
"""
mkvenv - QEMU pyvenv bootstrapping utility
usage: mkvenv [-h] command ...
QEMU pyvenv bootstrapping utility
options:
-h, --help show this help message and exit
Commands:
command Description
create create a venv
--------------------------------------------------
usage: mkvenv create [-h] target
positional arguments:
target Target directory to install virtual environment into.
options:
-h, --help show this help message and exit
"""
# Copyright (C) 2022-2023 Red Hat, Inc.
#
# Authors:
# John Snow <jsnow@redhat.com>
# Paolo Bonzini <pbonzini@redhat.com>
#
# This work is licensed under the terms of the GNU GPL, version 2 or
# later. See the COPYING file in the top-level directory.
import argparse
import logging
import os
from pathlib import Path
import subprocess
import sys
from types import SimpleNamespace
from typing import Any, Optional, Union
import venv
# Do not add any mandatory dependencies from outside the stdlib:
# This script *must* be usable standalone!
DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"]
logger = logging.getLogger("mkvenv")
class Ouch(RuntimeError):
"""An Exception class we can't confuse with a builtin."""
class QemuEnvBuilder(venv.EnvBuilder):
"""
An extension of venv.EnvBuilder for building QEMU's configure-time venv.
As of this commit, it does not yet do anything particularly
different than the standard venv-creation utility. The next several
commits will gradually change that in small commits that highlight
each feature individually.
Parameters for base class init:
- system_site_packages: bool = False
- clear: bool = False
- symlinks: bool = False
- upgrade: bool = False
- with_pip: bool = False
- prompt: Optional[str] = None
- upgrade_deps: bool = False (Since 3.9)
"""
def __init__(self, *args: Any, **kwargs: Any) -> None:
logger.debug("QemuEnvBuilder.__init__(...)")
super().__init__(*args, **kwargs)
# Make the context available post-creation:
self._context: Optional[SimpleNamespace] = None
def ensure_directories(self, env_dir: DirType) -> SimpleNamespace:
logger.debug("ensure_directories(env_dir=%s)", env_dir)
self._context = super().ensure_directories(env_dir)
return self._context
def get_value(self, field: str) -> str:
"""
Get a string value from the context namespace after a call to build.
For valid field names, see:
https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories
"""
ret = getattr(self._context, field)
assert isinstance(ret, str)
return ret
def make_venv( # pylint: disable=too-many-arguments
env_dir: Union[str, Path],
system_site_packages: bool = False,
clear: bool = True,
symlinks: Optional[bool] = None,
with_pip: bool = True,
) -> None:
"""
Create a venv using `QemuEnvBuilder`.
This is analogous to the `venv.create` module-level convenience
function that is part of the Python stdblib, except it uses
`QemuEnvBuilder` instead.
:param env_dir: The directory to create/install to.
:param system_site_packages:
Allow inheriting packages from the system installation.
:param clear: When True, fully remove any prior venv and files.
:param symlinks:
Whether to use symlinks to the target interpreter or not. If
left unspecified, it will use symlinks except on Windows to
match behavior with the "venv" CLI tool.
:param with_pip:
Whether to install "pip" binaries or not.
"""
logger.debug(
"%s: make_venv(env_dir=%s, system_site_packages=%s, "
"clear=%s, symlinks=%s, with_pip=%s)",
__file__,
str(env_dir),
system_site_packages,
clear,
symlinks,
with_pip,
)
if symlinks is None:
# Default behavior of standard venv CLI
symlinks = os.name != "nt"
builder = QemuEnvBuilder(
system_site_packages=system_site_packages,
clear=clear,
symlinks=symlinks,
with_pip=with_pip,
)
style = "non-isolated" if builder.system_site_packages else "isolated"
print(
f"mkvenv: Creating {style} virtual environment"
f" at '{str(env_dir)}'",
file=sys.stderr,
)
try:
logger.debug("Invoking builder.create()")
try:
builder.create(str(env_dir))
except SystemExit as exc:
# Some versions of the venv module raise SystemExit; *nasty*!
# We want the exception that prompted it. It might be a subprocess
# error that has output we *really* want to see.
logger.debug("Intercepted SystemExit from EnvBuilder.create()")
raise exc.__cause__ or exc.__context__ or exc
logger.debug("builder.create() finished")
except subprocess.CalledProcessError as exc:
logger.error("mkvenv subprocess failed:")
logger.error("cmd: %s", exc.cmd)
logger.error("returncode: %d", exc.returncode)
def _stringify(data: Union[str, bytes]) -> str:
if isinstance(data, bytes):
return data.decode()
return data
lines = []
if exc.stdout:
lines.append("========== stdout ==========")
lines.append(_stringify(exc.stdout))
lines.append("============================")
if exc.stderr:
lines.append("========== stderr ==========")
lines.append(_stringify(exc.stderr))
lines.append("============================")
if lines:
logger.error(os.linesep.join(lines))
raise Ouch("VENV creation subprocess failed.") from exc
# print the python executable to stdout for configure.
print(builder.get_value("env_exe"))
def _add_create_subcommand(subparsers: Any) -> None:
subparser = subparsers.add_parser("create", help="create a venv")
subparser.add_argument(
"target",
type=str,
action="store",
help="Target directory to install virtual environment into.",
)
def main() -> int:
"""CLI interface to make_qemu_venv. See module docstring."""
if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"):
# You're welcome.
logging.basicConfig(level=logging.DEBUG)
elif os.environ.get("V"):
logging.basicConfig(level=logging.INFO)
parser = argparse.ArgumentParser(
prog="mkvenv",
description="QEMU pyvenv bootstrapping utility",
)
subparsers = parser.add_subparsers(
title="Commands",
dest="command",
metavar="command",
help="Description",
)
_add_create_subcommand(subparsers)
args = parser.parse_args()
try:
if args.command == "create":
make_venv(
args.target,
system_site_packages=True,
clear=True,
)
logger.debug("mkvenv.py %s: exiting", args.command)
except Ouch as exc:
print("\n*** Ouch! ***\n", file=sys.stderr)
print(str(exc), "\n\n", file=sys.stderr)
return 1
except SystemExit:
raise
except: # pylint: disable=bare-except
logger.exception("mkvenv did not complete successfully:")
return 2
return 0
if __name__ == "__main__":
sys.exit(main())

View File

@ -103,6 +103,15 @@ ignore_missing_imports = True
[mypy-pygments]
ignore_missing_imports = True
[mypy-importlib.metadata]
ignore_missing_imports = True
[mypy-importlib_metadata]
ignore_missing_imports = True
[mypy-pkg_resources]
ignore_missing_imports = True
[pylint.messages control]
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this

View File

@ -1,2 +1,3 @@
#!/bin/sh -e
python3 -m flake8 qemu/
python3 -m flake8 scripts/

View File

@ -1,2 +1,3 @@
#!/bin/sh -e
python3 -m isort -c qemu/
python3 -m isort -c scripts/

View File

@ -1,2 +1,3 @@
#!/bin/sh -e
python3 -m mypy -p qemu
python3 -m mypy scripts/

View File

@ -1,3 +1,4 @@
#!/bin/sh -e
# See commit message for environment variable explainer.
SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pylint qemu/
SETUPTOOLS_USE_DISTUTILS=stdlib python3 -m pylint scripts/