diff --git a/ui/sdl.c b/ui/sdl.c
index 9a92b47f73..1563ee1965 100644
--- a/ui/sdl.c
+++ b/ui/sdl.c
@@ -583,10 +583,12 @@ static void sdl_refresh(DisplayState *ds)
                         gui_keysym = 1;
                         break;
                     case 0x16: /* 'u' key on US keyboard */
-                        scaling_active = 0;
-                        sdl_resize(ds);
-                        vga_hw_invalidate();
-                        vga_hw_update();
+                        if (scaling_active) {
+                            scaling_active = 0;
+                            sdl_resize(ds);
+                            vga_hw_invalidate();
+                            vga_hw_update();
+                        }
                         break;
                     case 0x02 ... 0x0a: /* '1' to '9' keys */
                         /* Reset the modifiers sent to the current console */