PR c++/55442 - memory-hog with highly recursive constexpr.

This testcase in the PR is extremely recursive, and therefore uses a huge
amount of memory on caching the results of individual calls.  We no longer
need to track all calls to catch infinite recursion, as we have other limits
on maximum depth and operations count.  So let's only cache a few calls at
the top level: 8 seems to be a reasonable compromise.

gcc/c-family/
	* c.opt (fconstexpr-loop-limit): New.
gcc/cp/
	* constexpr.c (push_cx_call_context): Return depth.
	(cxx_eval_call_expression): Don't cache past constexpr_cache_depth.

From-SVN: r272765

gcc/c-family/ChangeLog

@@ -1,3 +1,8 @@
+2019-06-26  Jason Merrill  <jason@redhat.com>
+
+	PR c++/55442 - memory-hog with highly recursive constexpr.
+	* c.opt (fconstexpr-loop-limit): New.
+
 2019-06-25  Jakub Jelinek  <jakub@redhat.com>
 
 	PR sanitizer/90954

gcc/c-family/c.opt

@@ -1424,6 +1424,10 @@ fconstexpr-depth=
 C++ ObjC++ Joined RejectNegative UInteger Var(max_constexpr_depth) Init(512)
 -fconstexpr-depth=<number>	Specify maximum constexpr recursion depth.
 
+fconstexpr-cache-depth=
+C++ ObjC++ Joined RejectNegative UInteger Var(constexpr_cache_depth) Init(8)
+-fconstexpr-cache-depth=<number>	Specify maximum constexpr recursion cache depth.
+
 fconstexpr-loop-limit=
 C++ ObjC++ Joined RejectNegative UInteger Var(constexpr_loop_limit) Init(262144)
 -fconstexpr-loop-limit=<number>	Specify maximum constexpr loop iteration count.

gcc/cp/ChangeLog

@@ -1,3 +1,9 @@
+2019-06-27  Jason Merrill  <jason@redhat.com>
+
+	PR c++/55442 - memory-hog with highly recursive constexpr.
+	* constexpr.c (push_cx_call_context): Return depth.
+	(cxx_eval_call_expression): Don't cache past constexpr_cache_depth.
+
 2019-06-27  Jan Hubicka  <jh@suse.cz>
 
 	* class.c (layout_class_type): Set TYPE_CXX_ODR_P for as-base

gcc/cp/constexpr.c

@@ -1447,16 +1447,17 @@ static vec<tree> call_stack;
 static int call_stack_tick;
 static int last_cx_error_tick;
 
-static bool
+static int
 push_cx_call_context (tree call)
 {
   ++call_stack_tick;
   if (!EXPR_HAS_LOCATION (call))
     SET_EXPR_LOCATION (call, input_location);
   call_stack.safe_push (call);
-  if (call_stack.length () > (unsigned) max_constexpr_depth)
+  int len = call_stack.length ();
+  if (len > max_constexpr_depth)
     return false;
-  return true;
+  return len;
 }
 
 static void
@@ -1587,7 +1588,7 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
   tree fun = get_function_named_in_call (t);
   constexpr_call new_call
     = { NULL, NULL, NULL, 0, ctx->manifestly_const_eval };
-  bool depth_ok;
+  int depth_ok;
 
   if (fun == NULL_TREE)
     return cxx_eval_internal_function (ctx, t, lval,
@@ -1791,14 +1792,20 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
       entry = *slot;
       if (entry == NULL)
 	{
-	  /* We need to keep a pointer to the entry, not just the slot, as the
+	  /* Only cache up to constexpr_cache_depth to limit memory use.  */
-	     slot can move in the call to cxx_eval_builtin_function_call.  */
+	  if (depth_ok < constexpr_cache_depth)
-	  *slot = entry = ggc_alloc<constexpr_call> ();
+	    {
-	  *entry = new_call;
+	      /* We need to keep a pointer to the entry, not just the slot, as
-	  fb.preserve ();
+		 the slot can move during evaluation of the body.  */
+	      *slot = entry = ggc_alloc<constexpr_call> ();
+	      *entry = new_call;
+	      fb.preserve ();
+	    }
 	}
-      /* Calls that are in progress have their result set to NULL,
+      /* Calls that are in progress have their result set to NULL, so that we
-	 so that we can detect circular dependencies.  */
+	 can detect circular dependencies.  Now that we only cache up to
+	 constexpr_cache_depth this won't catch circular dependencies that
+	 start deeper, but they'll hit the recursion or ops limit.  */
       else if (entry->result == NULL)
 	{
 	  if (!ctx->quiet)

gcc/doc/invoke.texi

@@ -209,8 +209,9 @@ in the following sections.
 @xref{C++ Dialect Options,,Options Controlling C++ Dialect}.
 @gccoptlist{-fabi-version=@var{n}  -fno-access-control @gol
 -faligned-new=@var{n}  -fargs-in-order=@var{n}  -fchar8_t  -fcheck-new @gol
--fconstexpr-depth=@var{n}  -fconstexpr-loop-limit=@var{n} @gol
+-fconstexpr-depth=@var{n}  -fconstexpr-cache-depth=@var{n} @gol
--fconstexpr-ops-limit=@var{n} -fno-elide-constructors @gol
+-fconstexpr-loop-limit=@var{n}  -fconstexpr-ops-limit=@var{n} @gol
+-fno-elide-constructors @gol
 -fno-enforce-eh-specs @gol
 -fno-gnu-keywords @gol
 -fno-implicit-templates @gol
@@ -2527,6 +2528,17 @@ to @var{n}.  A limit is needed to detect endless recursion during
 constant expression evaluation.  The minimum specified by the standard
 is 512.
 
+@item -fconstexpr-cache-depth=@var{n}
+@opindex fconstexpr-cache-depth
+Set the maximum level of nested evaluation depth for C++11 constexpr
+functions that will be cached to @var{n}.  This is a heuristic that
+trades off compilation speed (when the cache avoids repeated
+calculations) against memory consumption (when the cache grows very
+large from highly recursive evaluations).  The default is 8.  Very few
+users are likely to want to adjust it, but if your code does heavy
+constexpr calculations you might want to experiment to find which
+value works best for you.
+
 @item -fconstexpr-loop-limit=@var{n}
 @opindex fconstexpr-loop-limit
 Set the maximum number of iterations for a loop in C++14 constexpr functions