diff --git a/net/core/sock.c b/net/core/sock.c
index e4be66fb3bd5..08f16db46070 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1454,7 +1454,7 @@ void sk_destruct(struct sock *sk)
 
 static void __sk_free(struct sock *sk)
 {
-	if (unlikely(sock_diag_has_destroy_listeners(sk)))
+	if (unlikely(sock_diag_has_destroy_listeners(sk) && sk->sk_net_refcnt))
 		sock_diag_broadcast_destroy(sk);
 	else
 		sk_destruct(sk);