qemu-e2k/tests/qemu-iotests/common.rc

1031 lines
27 KiB
Plaintext
Raw Normal View History

#!/usr/bin/env bash
#
# Copyright (C) 2009 Red Hat, Inc.
# Copyright (c) 2000-2006 Silicon Graphics, Inc. All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
export LANG=C
PATH=".:$PATH"
HOSTOS=$(uname -s)
arch=$(uname -m)
[[ "$arch" =~ "ppc64" ]] && qemu_arch=ppc64 || qemu_arch="$arch"
# make sure we have a standard umask
umask 022
# bail out, setting up .notrun file
_notrun()
{
iotests: Write test output to TEST_DIR Drop the use of OUTPUT_DIR (test/qemu-iotests under the build directory), and instead write test output files (.out.bad, .notrun, and .casenotrun) to TEST_DIR. With this, the same test can be run concurrently without the separate instances interfering, because they will need separate TEST_DIRs anyway. Running the same test separately is useful when running the iotests with various format/protocol combinations in parallel, or when you just want to aggressively exercise a single test (e.g. when it fails only sporadically). Putting this output into TEST_DIR means that it will stick around for inspection after the test run is done (though running the same test in the same TEST_DIR will overwrite it, just as it used to be); but given that TEST_DIR is a scratch directory, it should be clear that users can delete all of its content at any point. (And if TEST_DIR is on tmpfs, it will just disappear on shutdown.) Contrarily, alternative approaches that would put these output files into OUTPUT_DIR with some prefix to differentiate between separate test runs might easily lead to cluttering OUTPUT_DIR. (This change means OUTPUT_DIR is no longer written to by the iotests, so we can drop its usage altogether.) Signed-off-by: Hanna Reitz <hreitz@redhat.com> Message-Id: <20220221172909.762858-1-hreitz@redhat.com> [hreitz: Simplified `Path(os.path.join(x, y))` to `Path(x, y)`, as suggested by Vladimir; and rebased on 9086c7639822b6 ("tests/qemu-iotests: Rework the checks and spots using GNU sed")] Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
2022-02-21 18:29:09 +01:00
echo "$*" >"$TEST_DIR/$seq.notrun"
echo "$seq not run: $*"
status=0
exit
}
if ! command -v gsed >/dev/null 2>&1; then
if sed --version 2>&1 | grep -v 'not GNU sed' | grep 'GNU sed' > /dev/null;
then
gsed()
{
sed "$@"
}
else
gsed()
{
_notrun "GNU sed not available"
}
fi
fi
dd()
{
if [ "$HOSTOS" == "Linux" ]
then
command dd --help | grep noxfer > /dev/null 2>&1
if [ "$?" -eq 0 ]
then
command dd status=noxfer $@
else
command dd $@
fi
else
command dd $@
fi
}
# poke_file 'test.img' 512 '\xff\xfe'
poke_file()
{
printf "$3" | dd "of=$1" bs=1 "seek=$2" conv=notrunc &>/dev/null
}
# poke_file_le $img_filename $offset $byte_width $value
# Example: poke_file_le "$TEST_IMG" 512 2 65534
poke_file_le()
{
local img=$1 ofs=$2 len=$3 val=$4 str=''
while ((len--)); do
str+=$(printf '\\x%02x' $((val & 0xff)))
val=$((val >> 8))
done
poke_file "$img" "$ofs" "$str"
}
# poke_file_be $img_filename $offset $byte_width $value
# Example: poke_file_be "$TEST_IMG" 512 2 65279
poke_file_be()
{
local img=$1 ofs=$2 len=$3 val=$4
local str=$(printf "%0$((len * 2))x\n" $val | sed 's/\(..\)/\\x\1/g')
poke_file "$img" "$ofs" "$str"
}
# peek_file_le 'test.img' 512 2 => 65534
peek_file_le()
{
local val=0 shift=0 byte
# coreutils' od --endian is not portable, so manually assemble bytes.
for byte in $(od -j"$2" -N"$3" -An -v -tu1 "$1"); do
val=$(( val | (byte << shift) ))
shift=$((shift + 8))
done
printf %llu $val
}
# peek_file_be 'test.img' 512 2 => 65279
peek_file_be()
{
local val=0 byte
# coreutils' od --endian is not portable, so manually assemble bytes.
for byte in $(od -j"$2" -N"$3" -An -v -tu1 "$1"); do
val=$(( (val << 8) | byte ))
done
printf %llu $val
}
# peek_file_raw 'test.img' 512 2 => '\xff\xfe'. Do not use if the raw data
# is likely to contain \0 or trailing \n.
peek_file_raw()
{
dd if="$1" bs=1 skip="$2" count="$3" status=none
}
_optstr_add()
{
if [ -n "$1" ]; then
echo "$1,$2"
else
echo "$2"
fi
}
# Set the variables to the empty string to turn Valgrind off
# for specific processes, e.g.
# $ VALGRIND_QEMU_IO= ./check -qcow2 -valgrind 015
: ${VALGRIND_QEMU_VM=$VALGRIND_QEMU}
: ${VALGRIND_QEMU_IMG=$VALGRIND_QEMU}
: ${VALGRIND_QEMU_IO=$VALGRIND_QEMU}
: ${VALGRIND_QEMU_NBD=$VALGRIND_QEMU}
: ${VALGRIND_QSD=$VALGRIND_QEMU}
# The Valgrind own parameters may be set with
# its environment variable VALGRIND_OPTS, e.g.
# $ VALGRIND_OPTS="--leak-check=yes" ./check -qcow2 -valgrind 015
_qemu_proc_exec()
{
local VALGRIND_LOGFILE="$1"
shift
if [[ "${VALGRIND_QEMU}" == "y" && "${NO_VALGRIND}" != "y" ]]; then
exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$@"
else
exec "$@"
fi
}
_qemu_proc_valgrind_log()
{
local VALGRIND_LOGFILE="$1"
local RETVAL="$2"
if [[ "${VALGRIND_QEMU}" == "y" && "${NO_VALGRIND}" != "y" ]]; then
if [ $RETVAL == 99 ]; then
cat "${VALGRIND_LOGFILE}"
fi
rm -f "${VALGRIND_LOGFILE}"
fi
}
_qemu_wrapper()
{
local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
(
if [ -n "${QEMU_NEED_PID}" ]; then
echo $BASHPID > "${QEMU_TEST_DIR}/qemu-${_QEMU_HANDLE}.pid"
fi
GDB=""
if [ -n "${GDB_OPTIONS}" ]; then
GDB="gdbserver ${GDB_OPTIONS}"
fi
VALGRIND_QEMU="${VALGRIND_QEMU_VM}" _qemu_proc_exec "${VALGRIND_LOGFILE}" \
$GDB "$QEMU_PROG" $QEMU_OPTIONS "$@"
)
RETVAL=$?
_qemu_proc_valgrind_log "${VALGRIND_LOGFILE}" $RETVAL
return $RETVAL
}
_qemu_img_wrapper()
{
local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
(
VALGRIND_QEMU="${VALGRIND_QEMU_IMG}" _qemu_proc_exec "${VALGRIND_LOGFILE}" \
"$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS "$@"
)
RETVAL=$?
_qemu_proc_valgrind_log "${VALGRIND_LOGFILE}" $RETVAL
return $RETVAL
}
_qemu_io_wrapper()
{
local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
local QEMU_IO_ARGS="$QEMU_IO_OPTIONS"
if [ "$IMGOPTSSYNTAX" = "true" ]; then
QEMU_IO_ARGS="--image-opts $QEMU_IO_ARGS"
if [ -n "$IMGKEYSECRET" ]; then
QEMU_IO_ARGS="--object secret,id=keysec0,data=$IMGKEYSECRET $QEMU_IO_ARGS"
fi
fi
(
VALGRIND_QEMU="${VALGRIND_QEMU_IO}" _qemu_proc_exec "${VALGRIND_LOGFILE}" \
"$QEMU_IO_PROG" $QEMU_IO_ARGS "$@"
)
RETVAL=$?
_qemu_proc_valgrind_log "${VALGRIND_LOGFILE}" $RETVAL
return $RETVAL
}
_qemu_nbd_wrapper()
{
local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
(
VALGRIND_QEMU="${VALGRIND_QEMU_NBD}" _qemu_proc_exec "${VALGRIND_LOGFILE}" \
"$QEMU_NBD_PROG" --pid-file="${QEMU_TEST_DIR}/qemu-nbd.pid" \
$QEMU_NBD_OPTIONS "$@"
)
RETVAL=$?
_qemu_proc_valgrind_log "${VALGRIND_LOGFILE}" $RETVAL
return $RETVAL
}
_qemu_storage_daemon_wrapper()
{
local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
(
if [ -n "${QSD_NEED_PID}" ]; then
echo $BASHPID > "${QEMU_TEST_DIR}/qemu-storage-daemon.pid"
fi
VALGRIND_QEMU="${VALGRIND_QSD}" _qemu_proc_exec "${VALGRIND_LOGFILE}" \
"$QSD_PROG" $QSD_OPTIONS "$@"
)
RETVAL=$?
_qemu_proc_valgrind_log "${VALGRIND_LOGFILE}" $RETVAL
return $RETVAL
}
# Valgrind bug #409141 https://bugs.kde.org/show_bug.cgi?id=409141
# Until valgrind 3.16+ is ubiquitous, we must work around a hang in
# valgrind when issuing sigkill. Disable valgrind for this invocation.
_NO_VALGRIND()
{
NO_VALGRIND="y" "$@"
}
export QEMU=_qemu_wrapper
export QEMU_IMG=_qemu_img_wrapper
export QEMU_IO=_qemu_io_wrapper
export QEMU_NBD=_qemu_nbd_wrapper
export QSD=_qemu_storage_daemon_wrapper
if [ "$IMGOPTSSYNTAX" = "true" ]; then
DRIVER="driver=$IMGFMT"
QEMU_IMG_EXTRA_ARGS="--image-opts $QEMU_IMG_EXTRA_ARGS"
if [ -n "$IMGKEYSECRET" ]; then
QEMU_IMG_EXTRA_ARGS="--object secret,id=keysec0,data=$IMGKEYSECRET $QEMU_IMG_EXTRA_ARGS"
fi
block: enable testing of LUKS driver with block I/O tests This adds support for testing the LUKS driver with the block I/O test framework. cd tests/qemu-io-tests ./check -luks A handful of test cases are modified to work with luks - 004 - whitelist luks format - 012 - use TEST_IMG_FILE instead of TEST_IMG for file ops - 048 - use TEST_IMG_FILE instead of TEST_IMG for file ops. don't assume extended image contents is all zeros, explicitly initialize with zeros Make file size smaller to avoid having to decrypt 1 GB of data. - 052 - don't assume initial image contents is all zeros, explicitly initialize with zeros - 100 - don't assume initial image contents is all zeros, explicitly initialize with zeros With this patch applied, the results are as follows: Passed: 001 002 003 004 005 008 009 010 011 012 021 032 043 047 048 049 052 087 100 134 143 Failed: 033 120 140 145 Skipped: 007 013 014 015 017 018 019 020 022 023 024 025 026 027 028 029 030 031 034 035 036 037 038 039 040 041 042 043 044 045 046 047 049 050 051 053 054 055 056 057 058 059 060 061 062 063 064 065 066 067 068 069 070 071 072 073 074 075 076 077 078 079 080 081 082 083 084 085 086 087 088 089 090 091 092 093 094 095 096 097 098 099 101 102 103 104 105 107 108 109 110 111 112 113 114 115 116 117 118 119 121 122 123 124 128 129 130 131 132 133 134 135 136 137 138 139 141 142 144 146 148 150 152 The reasons for the failed tests are: - 033 - needs adapting to use image opts syntax with blkdebug and test image in order to correctly set align property - 120 - needs adapting to use correct -drive syntax for luks - 140 - needs adapting to use correct -drive syntax for luks - 145 - needs adapting to use correct -drive syntax for luks The vast majority of skipped tests are exercising code that is qcow2 specific, though a couple could probably be usefully enabled for luks too. Signed-off-by: Daniel P. Berrange <berrange@redhat.com> Message-id: 1462896689-18450-4-git-send-email-berrange@redhat.com Signed-off-by: Max Reitz <mreitz@redhat.com>
2016-05-10 18:11:29 +02:00
if [ "$IMGFMT" = "luks" ]; then
DRIVER="$DRIVER,key-secret=keysec0"
fi
if [ "$IMGPROTO" = "file" ]; then
TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
TEST_IMG="$DRIVER,file.filename=$TEST_DIR/t.$IMGFMT"
elif [ "$IMGPROTO" = "nbd" ]; then
TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
TEST_IMG="$DRIVER,file.driver=nbd,file.type=unix"
TEST_IMG="$TEST_IMG,file.path=$SOCK_DIR/nbd"
elif [ "$IMGPROTO" = "fuse" ]; then
TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
TEST_IMG="$DRIVER,file.filename=$SOCK_DIR/fuse-t.$IMGFMT"
elif [ "$IMGPROTO" = "ssh" ]; then
TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
TEST_IMG="$DRIVER,file.driver=ssh,file.host=127.0.0.1,file.path=$TEST_IMG_FILE"
elif [ "$IMGPROTO" = "nfs" ]; then
TEST_DIR="$DRIVER,file.driver=nfs,file.filename=nfs://127.0.0.1/$TEST_DIR"
TEST_IMG=$TEST_DIR/t.$IMGFMT
else
TEST_IMG="$DRIVER,file.driver=$IMGPROTO,file.filename=$TEST_DIR/t.$IMGFMT"
fi
else
QEMU_IMG_EXTRA_ARGS=
if [ "$IMGPROTO" = "file" ]; then
TEST_IMG=$TEST_DIR/t.$IMGFMT
elif [ "$IMGPROTO" = "nbd" ]; then
TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
TEST_IMG="nbd+unix:///?socket=$SOCK_DIR/nbd"
elif [ "$IMGPROTO" = "fuse" ]; then
TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
TEST_IMG="$SOCK_DIR/fuse-t.$IMGFMT"
elif [ "$IMGPROTO" = "ssh" ]; then
TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
REMOTE_TEST_DIR="ssh://\\($USER@\\)\\?127.0.0.1\\(:[0-9]\\+\\)\\?$TEST_DIR"
TEST_IMG="ssh://127.0.0.1$TEST_IMG_FILE"
elif [ "$IMGPROTO" = "nfs" ]; then
TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
REMOTE_TEST_DIR="nfs://127.0.0.1$TEST_DIR"
TEST_IMG="nfs://127.0.0.1$TEST_IMG_FILE"
else
TEST_IMG=$IMGPROTO:$TEST_DIR/t.$IMGFMT
fi
fi
iotests: Let _make_test_img guess $TEST_IMG_FILE When most iotests want to create a test image that is named differently from the default $TEST_IMG, they do something like this: TEST_IMG="$TEST_IMG.base" _make_test_img $options This works fine with the "file" protocol, but not so much for anything else: _make_test_img tries to create an image under $TEST_IMG_FILE first, and only under $TEST_IMG if the former is not set; and on everything but "file", $TEST_IMG_FILE is set. There are two ways we can fix this: First, we could make all tests adjust not only TEST_IMG, but also TEST_IMG_FILE if that is present (e.g. with something like _set_test_img_suffix $suffix that would affect not only TEST_IMG but also TEST_IMG_FILE, if necessary). This is a pretty clean solution, and this is maybe what we should have done from the start. But it would also require changes to most existing bash tests. So the alternative is this: Let _make_test_img see whether $TEST_IMG_FILE still points to the original value. If so, it is possible that the caller has adjusted $TEST_IMG but not $TEST_IMG_FILE. In such a case, we can (for most protocols) derive the corresponding $TEST_IMG_FILE value from $TEST_IMG value and thus work around what technically is the caller misbehaving. This second solution is less clean, but it is robust against people keeping their old habit of adjusting TEST_IMG only, and requires much less changes. So this patch implements it. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Message-Id: <20201027190600.192171-15-mreitz@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com>
2020-10-27 20:05:54 +01:00
ORIG_TEST_IMG_FILE=$TEST_IMG_FILE
ORIG_TEST_IMG="$TEST_IMG"
FUSE_PIDS=()
FUSE_EXPORTS=()
if [ -z "$TEST_DIR" ]; then
TEST_DIR=$PWD/scratch
fi
QEMU_TEST_DIR="${TEST_DIR}"
if [ ! -e "$TEST_DIR" ]; then
mkdir "$TEST_DIR"
fi
if [ ! -d "$TEST_DIR" ]; then
echo "common.rc: Error: \$TEST_DIR ($TEST_DIR) is not a directory"
exit 1
fi
if [ -z "$REMOTE_TEST_DIR" ]; then
REMOTE_TEST_DIR="$TEST_DIR"
fi
if [ ! -d "$SAMPLE_IMG_DIR" ]; then
echo "common.rc: Error: \$SAMPLE_IMG_DIR ($SAMPLE_IMG_DIR) is not a directory"
exit 1
fi
_use_sample_img()
{
SAMPLE_IMG_FILE="${1%\.bz2}"
TEST_IMG="$TEST_DIR/$SAMPLE_IMG_FILE"
bzcat "$SAMPLE_IMG_DIR/$1" > "$TEST_IMG"
if [ $? -ne 0 ]
then
echo "_use_sample_img error, cannot extract '$SAMPLE_IMG_DIR/$1'"
exit 1
fi
}
_stop_nbd_server()
{
if [ -f "${QEMU_TEST_DIR}/qemu-nbd.pid" ]; then
local QEMU_NBD_PID
read QEMU_NBD_PID < "${QEMU_TEST_DIR}/qemu-nbd.pid"
kill ${QEMU_NBD_PID}
rm -f "${QEMU_TEST_DIR}/qemu-nbd.pid" "$SOCK_DIR/nbd"
fi
}
# Gets the data_file value from IMGOPTS and replaces the '$TEST_IMG'
# pattern by '$1'
# Caution: The replacement is done with sed, so $1 must be escaped
# properly. (The delimiter is '#'.)
_get_data_file()
{
if ! echo "$IMGOPTS" | grep -q 'data_file='; then
return 1
fi
echo "$IMGOPTS" | sed -e 's/.*data_file=\([^,]*\).*/\1/' \
| sed -e "s#\\\$TEST_IMG#$1#"
}
iotests: Let _make_test_img guess $TEST_IMG_FILE When most iotests want to create a test image that is named differently from the default $TEST_IMG, they do something like this: TEST_IMG="$TEST_IMG.base" _make_test_img $options This works fine with the "file" protocol, but not so much for anything else: _make_test_img tries to create an image under $TEST_IMG_FILE first, and only under $TEST_IMG if the former is not set; and on everything but "file", $TEST_IMG_FILE is set. There are two ways we can fix this: First, we could make all tests adjust not only TEST_IMG, but also TEST_IMG_FILE if that is present (e.g. with something like _set_test_img_suffix $suffix that would affect not only TEST_IMG but also TEST_IMG_FILE, if necessary). This is a pretty clean solution, and this is maybe what we should have done from the start. But it would also require changes to most existing bash tests. So the alternative is this: Let _make_test_img see whether $TEST_IMG_FILE still points to the original value. If so, it is possible that the caller has adjusted $TEST_IMG but not $TEST_IMG_FILE. In such a case, we can (for most protocols) derive the corresponding $TEST_IMG_FILE value from $TEST_IMG value and thus work around what technically is the caller misbehaving. This second solution is less clean, but it is robust against people keeping their old habit of adjusting TEST_IMG only, and requires much less changes. So this patch implements it. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Message-Id: <20201027190600.192171-15-mreitz@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com>
2020-10-27 20:05:54 +01:00
# Translate a $TEST_IMG to its corresponding $TEST_IMG_FILE for
# different protocols
_test_img_to_test_img_file()
{
case "$IMGPROTO" in
file)
echo "$1"
;;
fuse)
echo "$1" | sed -e "s#$SOCK_DIR/fuse-#$TEST_DIR/#"
;;
iotests: Let _make_test_img guess $TEST_IMG_FILE When most iotests want to create a test image that is named differently from the default $TEST_IMG, they do something like this: TEST_IMG="$TEST_IMG.base" _make_test_img $options This works fine with the "file" protocol, but not so much for anything else: _make_test_img tries to create an image under $TEST_IMG_FILE first, and only under $TEST_IMG if the former is not set; and on everything but "file", $TEST_IMG_FILE is set. There are two ways we can fix this: First, we could make all tests adjust not only TEST_IMG, but also TEST_IMG_FILE if that is present (e.g. with something like _set_test_img_suffix $suffix that would affect not only TEST_IMG but also TEST_IMG_FILE, if necessary). This is a pretty clean solution, and this is maybe what we should have done from the start. But it would also require changes to most existing bash tests. So the alternative is this: Let _make_test_img see whether $TEST_IMG_FILE still points to the original value. If so, it is possible that the caller has adjusted $TEST_IMG but not $TEST_IMG_FILE. In such a case, we can (for most protocols) derive the corresponding $TEST_IMG_FILE value from $TEST_IMG value and thus work around what technically is the caller misbehaving. This second solution is less clean, but it is robust against people keeping their old habit of adjusting TEST_IMG only, and requires much less changes. So this patch implements it. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Message-Id: <20201027190600.192171-15-mreitz@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com>
2020-10-27 20:05:54 +01:00
nfs)
echo "$1" | sed -e "s#nfs://127.0.0.1##"
;;
ssh)
echo "$1" | \
sed -e "s#ssh://\\($USER@\\)\\?127.0.0.1\\(:[0-9]\\+\\)\\?##"
;;
*)
return 1
;;
esac
}
_make_test_img()
{
# extra qemu-img options can be added by tests
# at least one argument (the image size) needs to be added
local extra_img_options=""
local optstr=""
local img_name=""
local use_backing=0
local backing_file=""
local object_options=""
local opts_param=false
local misc_params=()
if [[ $IMGPROTO == fuse && $TEST_IMG == $SOCK_DIR/fuse-* ]]; then
# The caller may be trying to overwrite an existing image
_rm_test_img "$TEST_IMG"
fi
iotests: Let _make_test_img guess $TEST_IMG_FILE When most iotests want to create a test image that is named differently from the default $TEST_IMG, they do something like this: TEST_IMG="$TEST_IMG.base" _make_test_img $options This works fine with the "file" protocol, but not so much for anything else: _make_test_img tries to create an image under $TEST_IMG_FILE first, and only under $TEST_IMG if the former is not set; and on everything but "file", $TEST_IMG_FILE is set. There are two ways we can fix this: First, we could make all tests adjust not only TEST_IMG, but also TEST_IMG_FILE if that is present (e.g. with something like _set_test_img_suffix $suffix that would affect not only TEST_IMG but also TEST_IMG_FILE, if necessary). This is a pretty clean solution, and this is maybe what we should have done from the start. But it would also require changes to most existing bash tests. So the alternative is this: Let _make_test_img see whether $TEST_IMG_FILE still points to the original value. If so, it is possible that the caller has adjusted $TEST_IMG but not $TEST_IMG_FILE. In such a case, we can (for most protocols) derive the corresponding $TEST_IMG_FILE value from $TEST_IMG value and thus work around what technically is the caller misbehaving. This second solution is less clean, but it is robust against people keeping their old habit of adjusting TEST_IMG only, and requires much less changes. So this patch implements it. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Message-Id: <20201027190600.192171-15-mreitz@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com>
2020-10-27 20:05:54 +01:00
if [ -z "$TEST_IMG_FILE" ]; then
img_name=$TEST_IMG
iotests: Let _make_test_img guess $TEST_IMG_FILE When most iotests want to create a test image that is named differently from the default $TEST_IMG, they do something like this: TEST_IMG="$TEST_IMG.base" _make_test_img $options This works fine with the "file" protocol, but not so much for anything else: _make_test_img tries to create an image under $TEST_IMG_FILE first, and only under $TEST_IMG if the former is not set; and on everything but "file", $TEST_IMG_FILE is set. There are two ways we can fix this: First, we could make all tests adjust not only TEST_IMG, but also TEST_IMG_FILE if that is present (e.g. with something like _set_test_img_suffix $suffix that would affect not only TEST_IMG but also TEST_IMG_FILE, if necessary). This is a pretty clean solution, and this is maybe what we should have done from the start. But it would also require changes to most existing bash tests. So the alternative is this: Let _make_test_img see whether $TEST_IMG_FILE still points to the original value. If so, it is possible that the caller has adjusted $TEST_IMG but not $TEST_IMG_FILE. In such a case, we can (for most protocols) derive the corresponding $TEST_IMG_FILE value from $TEST_IMG value and thus work around what technically is the caller misbehaving. This second solution is less clean, but it is robust against people keeping their old habit of adjusting TEST_IMG only, and requires much less changes. So this patch implements it. Signed-off-by: Max Reitz <mreitz@redhat.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Message-Id: <20201027190600.192171-15-mreitz@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com>
2020-10-27 20:05:54 +01:00
elif [ "$IMGOPTSSYNTAX" != "true" -a \
"$TEST_IMG_FILE" = "$ORIG_TEST_IMG_FILE" ]; then
# Handle cases of tests only updating TEST_IMG, but not TEST_IMG_FILE
img_name=$(_test_img_to_test_img_file "$TEST_IMG")
if [ "$?" != 0 ]; then
img_name=$TEST_IMG_FILE
fi
else
# $TEST_IMG_FILE is not the default value, so it definitely has been
# modified by the test
img_name=$TEST_IMG_FILE
fi
if [ -n "$IMGOPTS" ]; then
imgopts_expanded=$(echo "$IMGOPTS" | sed -e "s#\\\$TEST_IMG#$img_name#")
optstr=$(_optstr_add "$optstr" "$imgopts_expanded")
fi
if [ -n "$IMGKEYSECRET" ]; then
object_options="--object secret,id=keysec0,data=$IMGKEYSECRET"
optstr=$(_optstr_add "$optstr" "key-secret=keysec0")
fi
for param; do
if [ "$use_backing" = "1" -a -z "$backing_file" ]; then
backing_file=$param
continue
elif $opts_param; then
optstr=$(_optstr_add "$optstr" "$param")
opts_param=false
continue
fi
case "$param" in
-b)
use_backing=1
;;
-o)
opts_param=true
;;
--no-opts)
optstr=""
;;
*)
misc_params=("${misc_params[@]}" "$param")
;;
esac
done
if [ \( "$IMGFMT" = "qcow2" -o "$IMGFMT" = "qed" \) -a -n "$CLUSTER_SIZE" ]; then
optstr=$(_optstr_add "$optstr" "cluster_size=$CLUSTER_SIZE")
fi
if [ -n "$optstr" ]; then
extra_img_options="-o $optstr $extra_img_options"
fi
if [ $IMGPROTO = "nbd" ]; then
_stop_nbd_server
fi
# XXX(hch): have global image options?
(
if [ $use_backing = 1 ]; then
$QEMU_IMG create $object_options -f $IMGFMT $extra_img_options -b "$backing_file" "$img_name" "${misc_params[@]}" 2>&1
else
$QEMU_IMG create $object_options -f $IMGFMT $extra_img_options "$img_name" "${misc_params[@]}" 2>&1
fi
) | _filter_img_create
# Start an NBD server on the image file, which is what we'll be talking to.
# Once NBD gains resize support, we may also want to use -f raw at the
# server and interpret format over NBD, but for now, the format is
# interpreted at the server and raw data sent over NBD.
if [ $IMGPROTO = "nbd" ]; then
# Pass a sufficiently high number to -e that should be enough for all
# tests
eval "$QEMU_NBD -v -t -k '$SOCK_DIR/nbd' -f $IMGFMT -e 42 -x '' $TEST_IMG_FILE >/dev/null &"
sleep 1 # FIXME: qemu-nbd needs to be listening before we continue
fi
if [ $IMGPROTO = "fuse" -a -f "$img_name" ]; then
local export_mp
local pid
local pidfile
local timeout
export_mp=$(echo "$img_name" | sed -e "s#$TEST_DIR/#$SOCK_DIR/fuse-#")
if ! echo "$export_mp" | grep -q "^$SOCK_DIR"; then
echo 'Cannot use FUSE exports with images outside of TEST_DIR' >&2
return 1
fi
touch "$export_mp"
rm -f "$SOCK_DIR/fuse-output"
# Usually, users would export formatted nodes. But we present fuse as a
# protocol-level driver here, so we have to leave the format to the
# client.
export/fuse: Add allow-other option Without the allow_other mount option, no user (not even root) but the one who started qemu/the storage daemon can access the export. Allow users to configure the export such that such accesses are possible. While allow_other is probably what users want, we cannot make it an unconditional default, because passing it is only possible (for non-root users) if the global fuse.conf configuration file allows it. Thus, the default is an 'auto' mode, in which we first try with allow_other, and then fall back to without. FuseExport.allow_other reports whether allow_other was actually used as a mount option or not. Currently, this information is not used, but a future patch will let this field decide whether e.g. an export's UID and GID can be changed through chmod. One notable thing about 'auto' mode is that libfuse may print error messages directly to stderr, and so may fusermount (which it executes). Our export code cannot really filter or hide them. Therefore, if 'auto' fails its first attempt and has to fall back, fusermount will print an error message that mounting with allow_other failed. This behavior necessitates a change to iotest 308, namely we need to filter out this error message (because if the first attempt at mounting with allow_other succeeds, there will be no such message). Furthermore, common.rc's _make_test_img should use allow-other=off for FUSE exports, because iotests generally do not need to access images from other users, so allow-other=on or allow-other=auto have no advantage. OTOH, allow-other=on will not work on systems where user_allow_other is disabled, and with allow-other=auto, we get said error message that we would need to filter out again. Just disabling allow-other is simplest. Signed-off-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210625142317.271673-3-mreitz@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com>
2021-06-25 16:23:13 +02:00
# Switch off allow-other, because in general we do not need it for
# iotests. The default allow-other=auto has the downside of printing a
# fusermount error on its first attempt if allow_other is not
# permissible, which we would need to filter.
QSD_NEED_PID=y $QSD \
--blockdev file,node-name=export-node,filename=$img_name,discard=unmap \
export/fuse: Add allow-other option Without the allow_other mount option, no user (not even root) but the one who started qemu/the storage daemon can access the export. Allow users to configure the export such that such accesses are possible. While allow_other is probably what users want, we cannot make it an unconditional default, because passing it is only possible (for non-root users) if the global fuse.conf configuration file allows it. Thus, the default is an 'auto' mode, in which we first try with allow_other, and then fall back to without. FuseExport.allow_other reports whether allow_other was actually used as a mount option or not. Currently, this information is not used, but a future patch will let this field decide whether e.g. an export's UID and GID can be changed through chmod. One notable thing about 'auto' mode is that libfuse may print error messages directly to stderr, and so may fusermount (which it executes). Our export code cannot really filter or hide them. Therefore, if 'auto' fails its first attempt and has to fall back, fusermount will print an error message that mounting with allow_other failed. This behavior necessitates a change to iotest 308, namely we need to filter out this error message (because if the first attempt at mounting with allow_other succeeds, there will be no such message). Furthermore, common.rc's _make_test_img should use allow-other=off for FUSE exports, because iotests generally do not need to access images from other users, so allow-other=on or allow-other=auto have no advantage. OTOH, allow-other=on will not work on systems where user_allow_other is disabled, and with allow-other=auto, we get said error message that we would need to filter out again. Just disabling allow-other is simplest. Signed-off-by: Max Reitz <mreitz@redhat.com> Message-Id: <20210625142317.271673-3-mreitz@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com>
2021-06-25 16:23:13 +02:00
--export fuse,id=fuse-export,node-name=export-node,mountpoint="$export_mp",writable=on,growable=on,allow-other=off \
&
pidfile="$QEMU_TEST_DIR/qemu-storage-daemon.pid"
# Wait for the PID file
while [ ! -f "$pidfile" ]; do
sleep 0.5
done
pid=$(cat "$pidfile")
rm -f "$pidfile"
FUSE_PIDS+=($pid)
FUSE_EXPORTS+=("$export_mp")
fi
}
_rm_test_img()
{
local img=$1
if [[ $IMGPROTO == fuse && $img == $SOCK_DIR/fuse-* ]]; then
# Drop a FUSE export
local df_output
local i
local image_file
local index=''
local timeout
for i in "${!FUSE_EXPORTS[@]}"; do
if [ "${FUSE_EXPORTS[i]}" = "$img" ]; then
index=$i
break
fi
done
if [ -z "$index" ]; then
# Probably gone already
return 0
fi
kill "${FUSE_PIDS[index]}"
# Wait until the mount is gone
timeout=10 # *0.5 s
while true; do
# Will show the mount point; if the mount is still there,
# it will be $img.
df_output=$(df "$img" 2>/dev/null)
# But df may also show an error ("Transpoint endpoint not
# connected"), so retry in such cases
if [ -n "$df_output" ]; then
if ! echo "$df_output" | grep -q "$img"; then
break
fi
fi
sleep 0.5
timeout=$((timeout - 1))
if [ "$timeout" = 0 ]; then
echo 'Failed to take down FUSE export' >&2
return 1
fi
done
rm -f "$img"
unset "FUSE_PIDS[$index]"
unset "FUSE_EXPORTS[$index]"
image_file=$(echo "$img" | sed -e "s#$SOCK_DIR/fuse-#$TEST_DIR/#")
_rm_test_img "$image_file"
return
fi
if [ "$IMGFMT" = "vmdk" ]; then
# Remove all the extents for vmdk
"$QEMU_IMG" info "$img" 2>/dev/null | grep 'filename:' | cut -f 2 -d: \
| xargs -I {} rm -f "{}"
elif [ "$IMGFMT" = "qcow2" ]; then
# Remove external data file
if data_file=$(_get_data_file "$img"); then
rm -f "$data_file"
fi
fi
rm -f "$img"
}
_cleanup_test_img()
{
case "$IMGPROTO" in
nbd)
_stop_nbd_server
rm -f "$TEST_IMG_FILE"
;;
fuse)
local mp
for mp in "${FUSE_EXPORTS[@]}"; do
_rm_test_img "$mp"
done
FUSE_PIDS=()
FUSE_EXPORTS=()
;;
file)
_rm_test_img "$TEST_DIR/t.$IMGFMT"
_rm_test_img "$TEST_DIR/t.$IMGFMT.orig"
_rm_test_img "$TEST_DIR/t.$IMGFMT.base"
if [ -n "$SAMPLE_IMG_FILE" ]
then
rm -f "$TEST_DIR/$SAMPLE_IMG_FILE"
SAMPLE_IMG_FILE=
TEST_IMG="$ORIG_TEST_IMG"
fi
;;
rbd)
rbd --no-progress rm "$TEST_DIR/t.$IMGFMT" > /dev/null
;;
esac
}
_check_test_img()
{
(
if [ "$IMGOPTSSYNTAX" = "true" ]; then
$QEMU_IMG check $QEMU_IMG_EXTRA_ARGS "$@" "$TEST_IMG" 2>&1
else
$QEMU_IMG check "$@" -f $IMGFMT "$TEST_IMG" 2>&1
fi
) | _filter_testdir | _filter_qemu_img_check
# return real qemu_img check status, to analyze in
# _check_test_img_ignore_leaks
return ${PIPESTATUS[0]}
}
_check_test_img_ignore_leaks()
{
out=$(_check_test_img "$@")
status=$?
if [ $status = 3 ]; then
# This must correspond to success output in dump_human_image_check()
echo "No errors were found on the image."
return 0
fi
echo "$out"
return $status
}
_img_info()
{
if [[ "$1" == "--format-specific" ]]; then
local format_specific=1
shift
else
local format_specific=0
fi
discard=0
regex_json_spec_start='^ *"format-specific": \{'
$QEMU_IMG info $QEMU_IMG_EXTRA_ARGS "$@" "$TEST_IMG" 2>&1 | \
sed -e "s#$REMOTE_TEST_DIR#TEST_DIR#g" \
-e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
-e "s#$TEST_DIR#TEST_DIR#g" \
-e "s#$SOCK_DIR/fuse-#TEST_DIR/#g" \
-e "s#$IMGFMT#IMGFMT#g" \
-e 's/\(compression type: \)\(zlib\|zstd\)/\1COMPRESSION_TYPE/' \
-e "/^disk size:/ D" \
-e "/actual-size/ D" | \
while IFS='' read -r line; do
if [[ $format_specific == 1 ]]; then
discard=0
elif [[ $line == "Format specific information:" ]]; then
discard=1
elif [[ $line =~ $regex_json_spec_start ]]; then
discard=2
regex_json_spec_end="^${line%%[^ ]*}\\},? *$"
fi
if [[ $discard == 0 ]]; then
echo "$line"
elif [[ $discard == 1 && ! $line ]]; then
echo
discard=0
elif [[ $discard == 2 && $line =~ $regex_json_spec_end ]]; then
discard=0
fi
done
}
# bail out, setting up .casenotrun file
# The function _casenotrun() is used as a notifier. It is the
# caller's responsibility to make skipped a particular test.
#
_casenotrun()
{
iotests: Write test output to TEST_DIR Drop the use of OUTPUT_DIR (test/qemu-iotests under the build directory), and instead write test output files (.out.bad, .notrun, and .casenotrun) to TEST_DIR. With this, the same test can be run concurrently without the separate instances interfering, because they will need separate TEST_DIRs anyway. Running the same test separately is useful when running the iotests with various format/protocol combinations in parallel, or when you just want to aggressively exercise a single test (e.g. when it fails only sporadically). Putting this output into TEST_DIR means that it will stick around for inspection after the test run is done (though running the same test in the same TEST_DIR will overwrite it, just as it used to be); but given that TEST_DIR is a scratch directory, it should be clear that users can delete all of its content at any point. (And if TEST_DIR is on tmpfs, it will just disappear on shutdown.) Contrarily, alternative approaches that would put these output files into OUTPUT_DIR with some prefix to differentiate between separate test runs might easily lead to cluttering OUTPUT_DIR. (This change means OUTPUT_DIR is no longer written to by the iotests, so we can drop its usage altogether.) Signed-off-by: Hanna Reitz <hreitz@redhat.com> Message-Id: <20220221172909.762858-1-hreitz@redhat.com> [hreitz: Simplified `Path(os.path.join(x, y))` to `Path(x, y)`, as suggested by Vladimir; and rebased on 9086c7639822b6 ("tests/qemu-iotests: Rework the checks and spots using GNU sed")] Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
2022-02-21 18:29:09 +01:00
echo " [case not run] $*" >>"$TEST_DIR/$seq.casenotrun"
}
# just plain bail out
#
_fail()
{
iotests: Write test output to TEST_DIR Drop the use of OUTPUT_DIR (test/qemu-iotests under the build directory), and instead write test output files (.out.bad, .notrun, and .casenotrun) to TEST_DIR. With this, the same test can be run concurrently without the separate instances interfering, because they will need separate TEST_DIRs anyway. Running the same test separately is useful when running the iotests with various format/protocol combinations in parallel, or when you just want to aggressively exercise a single test (e.g. when it fails only sporadically). Putting this output into TEST_DIR means that it will stick around for inspection after the test run is done (though running the same test in the same TEST_DIR will overwrite it, just as it used to be); but given that TEST_DIR is a scratch directory, it should be clear that users can delete all of its content at any point. (And if TEST_DIR is on tmpfs, it will just disappear on shutdown.) Contrarily, alternative approaches that would put these output files into OUTPUT_DIR with some prefix to differentiate between separate test runs might easily lead to cluttering OUTPUT_DIR. (This change means OUTPUT_DIR is no longer written to by the iotests, so we can drop its usage altogether.) Signed-off-by: Hanna Reitz <hreitz@redhat.com> Message-Id: <20220221172909.762858-1-hreitz@redhat.com> [hreitz: Simplified `Path(os.path.join(x, y))` to `Path(x, y)`, as suggested by Vladimir; and rebased on 9086c7639822b6 ("tests/qemu-iotests: Rework the checks and spots using GNU sed")] Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
2022-02-21 18:29:09 +01:00
echo "$*" | tee -a "$TEST_DIR/$seq.full"
echo "(see $seq.full for details)"
status=1
exit 1
}
# tests whether $IMGFMT is one of the supported image formats for a test
#
_supported_fmt()
{
# "generic" is suitable for most image formats. For some formats it doesn't
# work, however (most notably read-only formats), so they can opt out by
# setting IMGFMT_GENERIC to false.
for f; do
if [ "$f" = "$IMGFMT" -o "$f" = "generic" -a "$IMGFMT_GENERIC" = "true" ]; then
if [ "$IMGFMT" = "luks" ]; then
_require_working_luks
fi
return
fi
done
_notrun "not suitable for this image format: $IMGFMT"
}
# tests whether $IMGFMT is one of the unsupported image format for a test
#
_unsupported_fmt()
{
for f; do
if [ "$f" = "$IMGFMT" ]; then
_notrun "not suitable for this image format: $IMGFMT"
fi
done
}
# tests whether $IMGPROTO is one of the supported image protocols for a test
#
_supported_proto()
{
for f; do
if [ "$f" = "$IMGPROTO" -o "$f" = "generic" ]; then
return
fi
done
_notrun "not suitable for this image protocol: $IMGPROTO"
}
# tests whether $IMGPROTO is specified as an unsupported image protocol for a test
#
_unsupported_proto()
{
for f; do
if [ "$f" = "$IMGPROTO" ]; then
_notrun "not suitable for this image protocol: $IMGPROTO"
return
fi
done
}
# tests whether the host OS is one of the supported OSes for a test
#
_supported_os()
{
for h
do
if [ "$h" = "$HOSTOS" ]
then
return
fi
done
_notrun "not suitable for this OS: $HOSTOS"
}
_supported_cache_modes()
{
for mode; do
if [ "$mode" = "$CACHEMODE" ]; then
return
fi
done
_notrun "not suitable for cache mode: $CACHEMODE"
}
# Check whether the filesystem supports O_DIRECT
_check_o_direct()
{
testfile="$TEST_DIR"/_check_o_direct
$QEMU_IMG create -f raw "$testfile" 1M > /dev/null
out=$($QEMU_IO -f raw -t none -c quit "$testfile" 2>&1)
rm -f "$testfile"
[[ "$out" != *"O_DIRECT"* ]]
}
_require_o_direct()
{
if ! _check_o_direct; then
_notrun "file system on $TEST_DIR does not support O_DIRECT"
fi
}
_check_cache_mode()
{
if [ $CACHEMODE == "none" ] || [ $CACHEMODE == "directsync" ]; then
_require_o_direct
fi
}
_check_cache_mode
# $1 - cache mode to use by default
# $2 - (optional) cache mode to use by default if O_DIRECT is not supported
_default_cache_mode()
{
if $CACHEMODE_IS_DEFAULT; then
if [ -z "$2" ] || _check_o_direct; then
CACHEMODE="$1"
else
CACHEMODE="$2"
fi
QEMU_IO="$QEMU_IO --cache $CACHEMODE"
_check_cache_mode
return
fi
}
_supported_aio_modes()
{
for mode; do
if [ "$mode" = "$AIOMODE" ]; then
return
fi
done
_notrun "not suitable for aio mode: $AIOMODE"
}
_default_aio_mode()
{
AIOMODE="$1"
QEMU_IO="$QEMU_IO --aio $1"
}
_unsupported_imgopts()
{
for bad_opt
do
# Add a space so tests can match for whitespace that marks the
# end of an option (\b or \> are not portable)
if echo "$IMGOPTS " | grep -q 2>/dev/null "$bad_opt"
then
_notrun "not suitable for image option: $bad_opt"
fi
done
}
# Caution: Overwrites $TEST_DIR/t.luks
_require_working_luks()
{
file="$TEST_DIR/t.luks"
output=$(
$QEMU_IMG create -f luks \
--object secret,id=sec0,data=hunter0 \
-o key-secret=sec0 \
-o iter-time=10 \
"$file" \
1M \
2>&1
)
status=$?
IMGFMT='luks' _rm_test_img "$file"
if [ $status != 0 ]; then
reason=$(echo "$output" | grep "$file:" | sed -e "s#.*$file: *##")
if [ -z "$reason" ]; then
reason="Failed to create a LUKS image"
fi
_notrun "$reason"
fi
}
# this test requires that a specified command (executable) exists
#
_require_command()
{
if [ "$1" = "QEMU" ]; then
c=$QEMU_PROG
elif [ "$1" = "QEMU_IMG" ]; then
c=$QEMU_IMG_PROG
elif [ "$1" = "QEMU_IO" ]; then
c=$QEMU_IO_PROG
elif [ "$1" = "QEMU_NBD" ]; then
c=$QEMU_NBD_PROG
else
eval c=\$$1
fi
[ -x "$c" ] || _notrun "$1 utility required, skipped this test"
}
# Check that a set of drivers has been whitelisted in the QEMU binary
#
_require_drivers()
{
available=$($QEMU -drive format=help | \
sed -e '/Supported formats:/!d' -e 's/Supported formats://')
for driver
do
if ! echo "$available" | grep -q " $driver\( \|$\)"; then
_notrun "$driver not available"
fi
done
}
# Check that we have a file system that allows huge (but very sparse) files
#
_require_large_file()
{
if ! truncate --size="$1" "$TEST_IMG"; then
_notrun "file system on $TEST_DIR does not support large enough files"
fi
rm "$TEST_IMG"
}
# Check that a set of devices is available in the QEMU binary
#
_require_devices()
{
available=$($QEMU -M none -device help | \
grep ^name | sed -e 's/^name "//' -e 's/".*$//')
for device
do
if ! echo "$available" | grep -q "$device" ; then
_notrun "$device not available"
fi
done
}
_require_one_device_of()
{
available=$($QEMU -M none -device help | \
grep ^name | sed -e 's/^name "//' -e 's/".*$//')
for device
do
if echo "$available" | grep -q "$device" ; then
return
fi
done
_notrun "$* not available"
}
_qcow2_dump_header()
{
if [[ "$1" == "--no-filter-compression" ]]; then
local filter_compression=0
shift
else
local filter_compression=1
fi
img="$1"
if [ -z "$img" ]; then
img="$TEST_IMG"
fi
if [[ $filter_compression == 0 ]]; then
$PYTHON qcow2.py "$img" dump-header
else
$PYTHON qcow2.py "$img" dump-header | _filter_qcow2_compression_type_bit
fi
}
# make sure this script returns success
true