diff --git a/hw/usb-wacom.c b/hw/usb-wacom.c
index fe052eb756..47f26cd0a3 100644
--- a/hw/usb-wacom.c
+++ b/hw/usb-wacom.c
@@ -160,6 +160,7 @@ static int usb_mouse_poll(USBWacomState *s, uint8_t *buf, int len)
     if (!s->mouse_grabbed) {
         s->eh_entry = qemu_add_mouse_event_handler(usb_mouse_event, s, 0,
                         "QEMU PenPartner tablet");
+        qemu_activate_mouse_event_handler(s->eh_entry);
         s->mouse_grabbed = 1;
     }
 
@@ -197,6 +198,7 @@ static int usb_wacom_poll(USBWacomState *s, uint8_t *buf, int len)
     if (!s->mouse_grabbed) {
         s->eh_entry = qemu_add_mouse_event_handler(usb_wacom_event, s, 1,
                         "QEMU PenPartner tablet");
+        qemu_activate_mouse_event_handler(s->eh_entry);
         s->mouse_grabbed = 1;
     }
 
@@ -334,8 +336,10 @@ static int usb_wacom_handle_control(USBDevice *dev, int request, int value,
         ret = 0;
         break;
     case WACOM_SET_REPORT:
-        qemu_remove_mouse_event_handler(s->eh_entry);
-        s->mouse_grabbed = 0;
+        if (s->mouse_grabbed) {
+            qemu_remove_mouse_event_handler(s->eh_entry);
+            s->mouse_grabbed = 0;
+        }
         s->mode = data[0];
         ret = 0;
         break;
@@ -397,7 +401,10 @@ static void usb_wacom_handle_destroy(USBDevice *dev)
 {
     USBWacomState *s = (USBWacomState *) dev;
 
-    qemu_remove_mouse_event_handler(s->eh_entry);
+    if (s->mouse_grabbed) {
+        qemu_remove_mouse_event_handler(s->eh_entry);
+        s->mouse_grabbed = 0;
+    }
 }
 
 static int usb_wacom_initfn(USBDevice *dev)