diff --git a/drivers/char/nozomi.c b/drivers/char/nozomi.c
index 7d73cd430340..2ad7d37afbd0 100644
--- a/drivers/char/nozomi.c
+++ b/drivers/char/nozomi.c
@@ -1651,10 +1651,10 @@ static void ntty_close(struct tty_struct *tty, struct file *file)
 
 	dc->open_ttys--;
 	port->count--;
-	tty_port_tty_set(port, NULL);
 
 	if (port->count == 0) {
 		DBG1("close: %d", nport->token_dl);
+		tty_port_tty_set(port, NULL);
 		spin_lock_irqsave(&dc->spin_mutex, flags);
 		dc->last_ier &= ~(nport->token_dl);
 		writew(dc->last_ier, dc->reg_ier);