block: wait for job callback in block_job_cancel_sync

The limitation on not having I/O after cancellation cannot really be
kept.  Even streaming has a very small race window where you could
cancel a job and have it report completion.  If this window is hit,
bdrv_change_backing_file() will yield and possibly cause accesses to
dangling pointers etc.

So, let's just assume that we cannot know exactly what will happen
after the coroutine has set busy to false.  We can set a very lax
condition:

- if we cancel the job, the coroutine won't set it to false again
(and hence will not call co_sleep_ns again).

- block_job_cancel_sync will wait for the coroutine to exit, which
pretty much ensures no race.

Instead, we track the coroutine that executes the job and put very
strict conditions on what to do while it is quiescent (busy = false).
First of all, the coroutine must never set busy = false while the job
has been cancelled.  Second, the coroutine can be reentered arbitrarily
while it is quiescent, so you cannot really do anything but co_sleep_ns at
that time.  This condition is obeyed by the block_job_sleep_ns function.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>

block.c

@@ -4238,6 +4238,9 @@ void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
 void block_job_cancel(BlockJob *job)
 {
     job->cancelled = true;
+    if (job->co && !job->busy) {
+        qemu_coroutine_enter(job->co, NULL);
+    }
 }
 
 bool block_job_is_cancelled(BlockJob *job)
@@ -4245,15 +4248,44 @@ bool block_job_is_cancelled(BlockJob *job)
     return job->cancelled;
 }
 
-void block_job_cancel_sync(BlockJob *job)
+struct BlockCancelData {
+    BlockJob *job;
+    BlockDriverCompletionFunc *cb;
+    void *opaque;
+    bool cancelled;
+    int ret;
+};
+
+static void block_job_cancel_cb(void *opaque, int ret)
 {
+    struct BlockCancelData *data = opaque;
+
+    data->cancelled = block_job_is_cancelled(data->job);
+    data->ret = ret;
+    data->cb(data->opaque, ret);
+}
+
+int block_job_cancel_sync(BlockJob *job)
+{
+    struct BlockCancelData data;
     BlockDriverState *bs = job->bs;
 
     assert(bs->job == job);
+
+    /* Set up our own callback to store the result and chain to
+     * the original callback.
+     */
+    data.job = job;
+    data.cb = job->cb;
+    data.opaque = job->opaque;
+    data.ret = -EINPROGRESS;
+    job->cb = block_job_cancel_cb;
+    job->opaque = &data;
     block_job_cancel(job);
-    while (bs->job != NULL && bs->job->busy) {
+    while (data.ret == -EINPROGRESS) {
         qemu_aio_wait();
     }
+    return (data.cancelled && data.ret == 0) ? -ECANCELED : data.ret;
 }
 
 void block_job_sleep_ns(BlockJob *job, QEMUClock *clock, int64_t ns)

block/stream.c

@@ -270,7 +270,6 @@ void stream_start(BlockDriverState *bs, BlockDriverState *base,
                   void *opaque, Error **errp)
 {
     StreamBlockJob *s;
-    Coroutine *co;
 
     s = block_job_create(&stream_job_type, bs, speed, cb, opaque, errp);
     if (!s) {
@@ -282,7 +281,7 @@ void stream_start(BlockDriverState *bs, BlockDriverState *base,
         pstrcpy(s->backing_file_id, sizeof(s->backing_file_id), base_id);
     }
 
-    co = qemu_coroutine_create(stream_run);
-    trace_stream_start(bs, base, s, co, opaque);
-    qemu_coroutine_enter(co, s);
+    s->common.co = qemu_coroutine_create(stream_run);
+    trace_stream_start(bs, base, s, s->common.co, opaque);
+    qemu_coroutine_enter(s->common.co, s);
 }

block_int.h

@@ -94,6 +94,12 @@ struct BlockJob {
     /** The block device on which the job is operating.  */
     BlockDriverState *bs;
 
+    /**
+     * The coroutine that executes the job.  If not NULL, it is
+     * reentered when busy is false and the job is cancelled.
+     */
+    Coroutine *co;
+
     /**
      * Set to true if the job should cancel itself.  The flag must
      * always be tested just before toggling the busy flag from false
@@ -418,8 +424,11 @@ bool block_job_is_cancelled(BlockJob *job);
  * immediately after #block_job_cancel_sync.  Users of block jobs
  * will usually protect the BlockDriverState objects with a reference
  * count, should this be a concern.
+ *
+ * Returns the return value from the job if the job actually completed
+ * during the call, or -ECANCELED if it was canceled.
  */
-void block_job_cancel_sync(BlockJob *job);
+int block_job_cancel_sync(BlockJob *job);
 
 /**
  * stream_start: