diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index fb2fb11fbb25..888fba457bb3 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -4618,10 +4618,8 @@ void cgroup_unload_subsys(struct cgroup_subsys *ss)
 	offline_css(ss, dummytop);
 	ss->active = 0;
 
-	if (ss->use_id) {
-		idr_remove_all(&ss->idr);
+	if (ss->use_id)
 		idr_destroy(&ss->idr);
-	}
 
 	/* deassign the subsys_id */
 	subsys[ss->subsys_id] = NULL;