diff --git a/tools/perf/ui/browsers/hists.c b/tools/perf/ui/browsers/hists.c
index ebd9d3afe2d6..c37e70ae0b20 100644
--- a/tools/perf/ui/browsers/hists.c
+++ b/tools/perf/ui/browsers/hists.c
@@ -1731,12 +1731,7 @@ do_annotate:
 			if (!notes->src)
 				continue;
 
-			/*
-			 * Don't let this be freed, say, by hists__decay_entry.
-			 */
-			he->used = true;
 			err = hist_entry__tui_annotate(he, evsel, hbt);
-			he->used = false;
 			/*
 			 * offer option to annotate the other branch source or target
 			 * (if they exists) when returning from annotate
diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c
index d9a6d35eda17..cc22b9158b93 100644
--- a/tools/perf/util/hist.c
+++ b/tools/perf/util/hist.c
@@ -263,15 +263,9 @@ void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel)
 	while (next) {
 		n = rb_entry(next, struct hist_entry, rb_node);
 		next = rb_next(&n->rb_node);
-		/*
-		 * We may be annotating this, for instance, so keep it here in
-		 * case some it gets new samples, we'll eventually free it when
-		 * the user stops browsing and it agains gets fully decayed.
-		 */
 		if (((zap_user && n->level == '.') ||
 		     (zap_kernel && n->level != '.') ||
-		     hists__decay_entry(hists, n)) &&
-		    !n->used) {
+		     hists__decay_entry(hists, n))) {
 			hists__delete_entry(hists, n);
 		}
 	}
diff --git a/tools/perf/util/sort.h b/tools/perf/util/sort.h
index c03e4ff8beff..9c01b83eebca 100644
--- a/tools/perf/util/sort.h
+++ b/tools/perf/util/sort.h
@@ -102,7 +102,6 @@ struct hist_entry {
 
 	bool			init_have_children;
 	char			level;
-	bool			used;
 	u8			filtered;
 	char			*srcline;
 	struct symbol		*parent;